diff --git a/lnbits/core/crud.py b/lnbits/core/crud.py index 33077ae8..ae984477 100644 --- a/lnbits/core/crud.py +++ b/lnbits/core/crud.py @@ -1,6 +1,6 @@ import datetime import json -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Literal, Optional from urllib.parse import urlparse from uuid import UUID, uuid4 @@ -9,7 +9,7 @@ import shortuuid from lnbits import bolt11 from lnbits.core.db import db from lnbits.core.models import WalletType -from lnbits.db import Connection, Database, Filters, Page +from lnbits.db import DB_TYPE, SQLITE, Connection, Database, Filters, Page from lnbits.extension_manager import InstallableExtension from lnbits.settings import ( AdminSettings, @@ -23,6 +23,7 @@ from .models import ( BalanceCheck, Payment, PaymentFilters, + PaymentHistoryPoint, TinyURL, User, Wallet, @@ -655,6 +656,79 @@ async def update_payment_extra( ) +async def update_pending_payments(wallet_id: str): + pending_payments = await get_payments( + wallet_id=wallet_id, + pending=True, + exclude_uncheckable=True, + ) + for payment in pending_payments: + await payment.check_status() + + +DateTrunc = Literal["hour", "day", "month"] +sqlite_formats = { + "hour": "%Y-%m-%d %H:00:00", + "day": "%Y-%m-%d 00:00:00", + "month": "%Y-%m-01 00:00:00", +} + + +async def get_payments_history( + wallet_id: Optional[str] = None, + group: DateTrunc = "day", + filters: Optional[Filters] = None, +) -> List[PaymentHistoryPoint]: + if not filters: + filters = Filters() + where = ["(pending = False OR amount < 0)"] + values = [] + if wallet_id: + where.append("wallet = ?") + values.append(wallet_id) + + if DB_TYPE == SQLITE and group in sqlite_formats: + date_trunc = f"strftime('{sqlite_formats[group]}', time, 'unixepoch')" + elif group in ("day", "hour", "month"): + date_trunc = f"date_trunc('{group}', time)" + else: + raise ValueError(f"Invalid group value: {group}") + + transactions = await db.fetchall( + f""" + SELECT {date_trunc} date, + SUM(CASE WHEN amount > 0 THEN amount ELSE 0 END) income, + SUM(CASE WHEN amount < 0 THEN abs(amount) + abs(fee) ELSE 0 END) spending + FROM apipayments + {filters.where(where)} + GROUP BY date + ORDER BY date DESC + """, + filters.values(values), + ) + if wallet_id: + wallet = await get_wallet(wallet_id) + if wallet: + balance = wallet.balance_msat + else: + raise ValueError("Unknown wallet") + else: + balance = await get_total_balance() + + # since we dont know the balance at the starting point, + # we take the current balance and walk backwards + results: list[PaymentHistoryPoint] = [] + for row in transactions: + results.insert( + 0, + PaymentHistoryPoint( + balance=balance, date=row[0], income=row[1], spending=row[2] + ), + ) + balance -= row.income - row.spending + return results + + async def delete_wallet_payment( checking_id: str, wallet_id: str, conn: Optional[Connection] = None ) -> None: diff --git a/lnbits/core/models.py b/lnbits/core/models.py index 49ab2108..3668aef0 100644 --- a/lnbits/core/models.py +++ b/lnbits/core/models.py @@ -258,6 +258,13 @@ class PaymentFilters(FilterModel): webhook_status: Optional[int] +class PaymentHistoryPoint(BaseModel): + date: datetime.datetime + income: int + spending: int + balance: int + + class BalanceCheck(BaseModel): wallet: str service: str diff --git a/lnbits/core/static/js/wallet.js b/lnbits/core/static/js/wallet.js index 67c67415..69f22871 100644 --- a/lnbits/core/static/js/wallet.js +++ b/lnbits/core/static/js/wallet.js @@ -3,45 +3,24 @@ Vue.component(VueQrcode.name, VueQrcode) Vue.use(VueQrcodeReader) -function generateChart(canvas, payments) { - var txs = [] - var n = 0 - var data = { - labels: [], - income: [], - outcome: [], - cumulative: [] - } - - _.each( - payments.filter(p => !p.pending).sort((a, b) => a.time - b.time), - tx => { - txs.push({ - hour: Quasar.utils.date.formatDate(tx.date, 'YYYY-MM-DDTHH:00'), - sat: tx.sat - }) +function generateChart(canvas, rawData) { + const data = rawData.reduce( + (previous, current) => { + previous.labels.push(current.date) + previous.income.push(current.income) + previous.spending.push(current.spending) + previous.cumulative.push(current.balance) + return previous + }, + { + labels: [], + income: [], + spending: [], + cumulative: [] } ) - _.each(_.groupBy(txs, 'hour'), (value, day) => { - var income = _.reduce( - value, - (memo, tx) => (tx.sat >= 0 ? memo + tx.sat : memo), - 0 - ) - var outcome = _.reduce( - value, - (memo, tx) => (tx.sat < 0 ? memo + Math.abs(tx.sat) : memo), - 0 - ) - n = n + income - outcome - data.labels.push(day) - data.income.push(income) - data.outcome.push(outcome) - data.cumulative.push(n) - }) - - new Chart(canvas.getContext('2d'), { + return new Chart(canvas.getContext('2d'), { type: 'bar', data: { labels: data.labels, @@ -64,7 +43,7 @@ function generateChart(canvas, payments) { backgroundColor: window.Color('rgb(76,175,80)').alpha(0.5).rgbString() // green }, { - data: data.outcome, + data: data.spending, type: 'bar', label: 'out', barPercentage: 0.75, @@ -85,7 +64,7 @@ function generateChart(canvas, payments) { { type: 'time', display: true, - offset: true, + //offset: true, time: { minUnit: 'hour', stepSize: 3 @@ -248,7 +227,14 @@ new Vue({ loading: false }, paymentsChart: { - show: false + show: false, + group: {value: 'hour', label: 'Hour'}, + groupOptions: [ + {value: 'month', label: 'Month'}, + {value: 'day', label: 'Day'}, + {value: 'hour', label: 'Hour'} + ], + instance: null }, disclaimerDialog: { show: false, @@ -301,9 +287,27 @@ new Vue({ }, showChart: function () { this.paymentsChart.show = true - this.$nextTick(() => { - generateChart(this.$refs.canvas, this.payments) - }) + LNbits.api + .request( + 'GET', + '/api/v1/payments/history?group=' + this.paymentsChart.group.value, + this.g.wallet.adminkey + ) + .then(response => { + this.$nextTick(() => { + if (this.paymentsChart.instance) { + this.paymentsChart.instance.destroy() + } + this.paymentsChart.instance = generateChart( + this.$refs.canvas, + response.data + ) + }) + }) + .catch(err => { + LNbits.utils.notifyApiError(err) + this.paymentsChart.show = false + }) }, focusInput(el) { this.$nextTick(() => this.$refs[el].focus()) @@ -803,6 +807,9 @@ new Vue({ watch: { payments: function () { this.fetchBalance() + }, + 'paymentsChart.group': function () { + this.showChart() } }, created: function () { diff --git a/lnbits/core/templates/core/wallet.html b/lnbits/core/templates/core/wallet.html index bb8e1154..534fe52e 100644 --- a/lnbits/core/templates/core/wallet.html +++ b/lnbits/core/templates/core/wallet.html @@ -827,6 +827,19 @@ +
+
Payments Chart
+ + +
+ diff --git a/lnbits/core/views/api.py b/lnbits/core/views/api.py index 98cafc62..9c6d9153 100644 --- a/lnbits/core/views/api.py +++ b/lnbits/core/views/api.py @@ -40,6 +40,8 @@ from lnbits.core.models import ( DecodePayment, Payment, PaymentFilters, + PaymentHistoryPoint, + Query, User, Wallet, WalletType, @@ -71,6 +73,7 @@ from lnbits.utils.exchange_rates import ( ) from ..crud import ( + DateTrunc, add_installed_extension, create_tinyurl, create_webpush_subscription, @@ -81,6 +84,7 @@ from ..crud import ( drop_extension_db, get_dbversions, get_payments, + get_payments_history, get_payments_paginated, get_standalone_payment, get_tinyurl, @@ -88,6 +92,7 @@ from ..crud import ( get_wallet_for_key, get_webpush_subscription, save_balance_check, + update_pending_payments, update_wallet, ) from ..services import ( @@ -155,16 +160,7 @@ async def api_payments( wallet: WalletTypeInfo = Depends(get_key_type), filters: Filters = Depends(parse_filters(PaymentFilters)), ): - pending_payments = await get_payments( - wallet_id=wallet.wallet.id, - pending=True, - exclude_uncheckable=True, - filters=filters, - ) - for payment in pending_payments: - await check_transaction_status( - wallet_id=payment.wallet_id, payment_hash=payment.payment_hash - ) + await update_pending_payments(wallet.wallet.id) return await get_payments( wallet_id=wallet.wallet.id, pending=True, @@ -173,6 +169,21 @@ async def api_payments( ) +@api_router.get( + "/api/v1/payments/history", + name="Get payments history", + response_model=List[PaymentHistoryPoint], + openapi_extra=generate_filter_params_openapi(PaymentFilters), +) +async def api_payments_history( + wallet: WalletTypeInfo = Depends(get_key_type), + group: DateTrunc = Query("day"), + filters: Filters[PaymentFilters] = Depends(parse_filters(PaymentFilters)), +): + await update_pending_payments(wallet.wallet.id) + return await get_payments_history(wallet.wallet.id, group, filters) + + @api_router.get( "/api/v1/payments/paginated", name="Payment List", @@ -185,16 +196,7 @@ async def api_payments_paginated( wallet: WalletTypeInfo = Depends(get_key_type), filters: Filters = Depends(parse_filters(PaymentFilters)), ): - pending = await get_payments_paginated( - wallet_id=wallet.wallet.id, - pending=True, - exclude_uncheckable=True, - filters=filters, - ) - for payment in pending.data: - await check_transaction_status( - wallet_id=payment.wallet_id, payment_hash=payment.payment_hash - ) + await update_pending_payments(wallet.wallet.id) page = await get_payments_paginated( wallet_id=wallet.wallet.id, pending=True, diff --git a/tests/conftest.py b/tests/conftest.py index 117c0560..6b9e1dbb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 import asyncio +from time import time import uvloop @@ -11,11 +12,16 @@ from fastapi.testclient import TestClient from httpx import AsyncClient from lnbits.app import create_app -from lnbits.core.crud import create_account, create_wallet, get_user +from lnbits.core.crud import ( + create_account, + create_wallet, + get_user, + update_payment_status, +) from lnbits.core.models import CreateInvoice from lnbits.core.services import update_wallet_balance from lnbits.core.views.api import api_payments_create_invoice -from lnbits.db import Database +from lnbits.db import DB_TYPE, SQLITE, Database from lnbits.settings import settings from tests.helpers import ( clean_database, @@ -173,6 +179,31 @@ async def real_invoice(): del invoice +@pytest_asyncio.fixture(scope="session") +async def fake_payments(client, adminkey_headers_from): + # Because sqlite only stores timestamps with milliseconds + # we have to wait a second to ensure a different timestamp than previous invoices + if DB_TYPE == SQLITE: + await asyncio.sleep(1) + ts = time() + + fake_data = [ + CreateInvoice(amount=10, memo="aaaa", out=False), + CreateInvoice(amount=100, memo="bbbb", out=False), + CreateInvoice(amount=1000, memo="aabb", out=False), + ] + + for invoice in fake_data: + response = await client.post( + "/api/v1/payments", headers=adminkey_headers_from, json=invoice.dict() + ) + assert response.is_success + await update_payment_status(response.json()["checking_id"], pending=False) + + params = {"time[ge]": ts, "time[le]": time()} + return fake_data, params + + @pytest_asyncio.fixture(scope="function") async def hold_invoice(): invoice = get_hold_invoice(100) diff --git a/tests/core/views/test_api.py b/tests/core/views/test_api.py index 2115b58d..47a37889 100644 --- a/tests/core/views/test_api.py +++ b/tests/core/views/test_api.py @@ -1,23 +1,21 @@ import asyncio import hashlib -from time import time import pytest from lnbits import bolt11 from lnbits.core.crud import get_standalone_payment, update_payment_details -from lnbits.core.models import Payment +from lnbits.core.models import CreateInvoice, Payment from lnbits.core.views.admin_api import api_auditor from lnbits.core.views.api import api_payment -from lnbits.db import DB_TYPE, SQLITE from lnbits.settings import settings from lnbits.wallets import get_wallet_class -from tests.conftest import CreateInvoice, api_payments_create_invoice from ...helpers import ( cancel_invoice, get_random_invoice_data, is_fake, + is_regtest, pay_real_invoice, settle_invoice, ) @@ -250,29 +248,13 @@ async def test_pay_invoice_adminkey(client, invoice, adminkey_headers_from): @pytest.mark.asyncio -async def test_get_payments(client, from_wallet, adminkey_headers_from): - # Because sqlite only stores timestamps with milliseconds we have to wait a second - # to ensure a different timestamp than previous invoices due to this limitation - # both payments (normal and paginated) are tested at the same time as they are - # almost identical anyways - if DB_TYPE == SQLITE: - await asyncio.sleep(1) - ts = time() - - fake_data = [ - CreateInvoice(amount=10, memo="aaaa"), - CreateInvoice(amount=100, memo="bbbb"), - CreateInvoice(amount=1000, memo="aabb"), - ] - - for invoice in fake_data: - await api_payments_create_invoice(invoice, from_wallet) +async def test_get_payments(client, adminkey_headers_from, fake_payments): + fake_data, filters = fake_payments async def get_payments(params: dict): - params["time[ge]"] = ts response = await client.get( "/api/v1/payments", - params=params, + params=filters | params, headers=adminkey_headers_from, ) assert response.status_code == 200 @@ -298,9 +280,14 @@ async def test_get_payments(client, from_wallet, adminkey_headers_from): payments = await get_payments({"amount[gt]": 10000}) assert len(payments) == 2 + +@pytest.mark.asyncio +async def test_get_payments_paginated(client, adminkey_headers_from, fake_payments): + fake_data, filters = fake_payments + response = await client.get( "/api/v1/payments/paginated", - params={"limit": 2, "time[ge]": ts}, + params=filters | {"limit": 2}, headers=adminkey_headers_from, ) assert response.status_code == 200 @@ -309,6 +296,38 @@ async def test_get_payments(client, from_wallet, adminkey_headers_from): assert paginated["total"] == len(fake_data) +@pytest.mark.asyncio +@pytest.mark.skipif( + is_regtest, reason="payments wont be confirmed rightaway in regtest" +) +async def test_get_payments_history(client, adminkey_headers_from, fake_payments): + fake_data, filters = fake_payments + + response = await client.get( + "/api/v1/payments/history", + params=filters, + headers=adminkey_headers_from, + ) + + assert response.status_code == 200 + data = response.json() + assert len(data) == 1 + assert data[0]["spending"] == sum( + payment.amount * 1000 for payment in fake_data if payment.out + ) + assert data[0]["income"] == sum( + payment.amount * 1000 for payment in fake_data if not payment.out + ) + + response = await client.get( + "/api/v1/payments/history?group=INVALID", + params=filters, + headers=adminkey_headers_from, + ) + + assert response.status_code == 400 + + # check POST /api/v1/payments/decode @pytest.mark.asyncio async def test_decode_invoice(client, invoice):