test: restructure tests (#2444)

unit, api, wallets
* only run test-api for migration

tests/regtest/conftest.py

@@ -0,0 +1,17 @@
+import pytest_asyncio
+
+from .helpers import get_hold_invoice, get_real_invoice
+
+
+@pytest_asyncio.fixture(scope="function")
+async def hold_invoice():
+    invoice = get_hold_invoice(100)
+    yield invoice
+    del invoice
+
+
+@pytest_asyncio.fixture(scope="function")
+async def real_invoice():
+    invoice = get_real_invoice(100)
+    yield {"bolt11": invoice["payment_request"]}
+    del invoice

tests/regtest/helpers.py

@@ -0,0 +1,130 @@
+import hashlib
+import json
+import os
+import time
+from subprocess import PIPE, Popen, TimeoutExpired
+from typing import Tuple
+
+from loguru import logger
+
+docker_lightning_cli = [
+    "docker",
+    "exec",
+    "lnbits-lnd-1-1",
+    "lncli",
+    "--network",
+    "regtest",
+    "--rpcserver=lnd-1",
+]
+
+docker_bitcoin_cli = [
+    "docker",
+    "exec",
+    "lnbits-bitcoind-1-1" "bitcoin-cli",
+    "-rpcuser=lnbits",
+    "-rpcpassword=lnbits",
+    "-regtest",
+]
+
+
+docker_lightning_unconnected_cli = [
+    "docker",
+    "exec",
+    "lnbits-lnd-2-1",
+    "lncli",
+    "--network",
+    "regtest",
+    "--rpcserver=lnd-2",
+]
+
+
+def run_cmd(cmd: list) -> str:
+    timeout = 20
+    process = Popen(cmd, stdout=PIPE, stderr=PIPE)
+
+    def process_communication(comm):
+        stdout, stderr = comm
+        output = stdout.decode("utf-8").strip()
+        error = stderr.decode("utf-8").strip()
+        return output, error
+
+    try:
+        now = time.time()
+        output, error = process_communication(process.communicate(timeout=timeout))
+        took = time.time() - now
+        logger.debug(f"ran command output: {output}, error: {error}, took: {took}s")
+        return output
+    except TimeoutExpired:
+        process.kill()
+        output, error = process_communication(process.communicate())
+        logger.error(f"timeout command: {cmd}, output: {output}, error: {error}")
+        raise
+
+
+def run_cmd_json(cmd: list) -> dict:
+    output = run_cmd(cmd)
+    try:
+        return json.loads(output) if output else {}
+    except json.decoder.JSONDecodeError:
+        logger.error(f"failed to decode json from cmd `{cmd}`: {output}")
+        raise
+
+
+def get_hold_invoice(sats: int) -> Tuple[str, dict]:
+    preimage = os.urandom(32)
+    preimage_hash = hashlib.sha256(preimage).hexdigest()
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["addholdinvoice", preimage_hash, str(sats)])
+    json = run_cmd_json(cmd)
+    return preimage.hex(), json
+
+
+def settle_invoice(preimage: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["settleinvoice", preimage])
+    return run_cmd(cmd)
+
+
+def cancel_invoice(preimage_hash: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["cancelinvoice", preimage_hash])
+    return run_cmd(cmd)
+
+
+def get_real_invoice(sats: int) -> dict:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["addinvoice", str(sats)])
+    return run_cmd_json(cmd)
+
+
+def pay_real_invoice(invoice: str) -> str:
+    cmd = docker_lightning_cli.copy()
+    cmd.extend(["payinvoice", "--force", invoice])
+    return run_cmd(cmd)
+
+
+def mine_blocks(blocks: int = 1) -> str:
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["-generate", str(blocks)])
+    return run_cmd(cmd)
+
+
+def get_unconnected_node_uri() -> str:
+    cmd = docker_lightning_unconnected_cli.copy()
+    cmd.append("getinfo")
+    info = run_cmd_json(cmd)
+    pubkey = info["identity_pubkey"]
+    return f"{pubkey}@lnd-2:9735"
+
+
+def create_onchain_address(address_type: str = "bech32") -> str:
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["getnewaddress", address_type])
+    return run_cmd(cmd)
+
+
+def pay_onchain(address: str, sats: int) -> str:
+    btc = sats * 0.00000001
+    cmd = docker_bitcoin_cli.copy()
+    cmd.extend(["sendtoaddress", address, str(btc)])
+    return run_cmd(cmd)

tests/regtest/test_real_invoice.py

@@ -0,0 +1,390 @@
+import asyncio
+import hashlib
+
+import pytest
+
+from lnbits import bolt11
+from lnbits.core.crud import get_standalone_payment, update_payment_details
+from lnbits.core.models import CreateInvoice, Payment
+from lnbits.core.services import fee_reserve_total
+from lnbits.core.views.admin_api import api_auditor
+from lnbits.core.views.payment_api import api_payment
+from lnbits.wallets import get_funding_source
+
+from ..helpers import is_fake, is_regtest
+from .helpers import (
+    cancel_invoice,
+    get_real_invoice,
+    pay_real_invoice,
+    settle_invoice,
+)
+
+
+async def get_node_balance_sats():
+    audit = await api_auditor()
+    return audit["node_balance_msats"] / 1000
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_real_invoice(
+    client, real_invoice, adminkey_headers_from, inkey_headers_from, from_wallet_ws
+):
+    prev_balance = await get_node_balance_sats()
+    response = await client.post(
+        "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+    assert len(invoice["payment_hash"]) == 64
+    assert len(invoice["checking_id"]) > 0
+
+    data = from_wallet_ws.receive_json()
+    assert "wallet_balance" in data
+    payment = Payment(**data["payment"])
+    assert payment.payment_hash == invoice["payment_hash"]
+
+    # check the payment status
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert payment_status["paid"]
+
+    funding_source = get_funding_source()
+    status = await funding_source.get_payment_status(invoice["payment_hash"])
+    assert status.paid
+
+    await asyncio.sleep(1)
+    balance = await get_node_balance_sats()
+    assert prev_balance - balance == 100
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_from):
+    prev_balance = await get_node_balance_sats()
+    create_invoice = CreateInvoice(out=False, amount=1000, memo="test")
+    response = await client.post(
+        "/api/v1/payments",
+        json=create_invoice.dict(),
+        headers=adminkey_headers_from,
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert not payment_status["paid"]
+
+    async def listen():
+        found_checking_id = False
+        async for checking_id in get_funding_source().paid_invoices_stream():
+            if checking_id == invoice["checking_id"]:
+                found_checking_id = True
+                return
+        assert found_checking_id
+
+    task = asyncio.create_task(listen())
+    await asyncio.sleep(1)
+    pay_real_invoice(invoice["payment_request"])
+    await asyncio.wait_for(task, timeout=10)
+
+    response = await client.get(
+        f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
+    )
+    assert response.status_code < 300
+    payment_status = response.json()
+    assert payment_status["paid"]
+
+    await asyncio.sleep(1)
+    balance = await get_node_balance_sats()
+    assert balance - prev_balance == create_invoice.amount
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_real_invoice_set_pending_and_check_state(
+    client, real_invoice, adminkey_headers_from, inkey_headers_from
+):
+    """
+    1. We create an invoice
+    2. We pay it
+    3. We verify that the inoice was paid
+    4. We set the invoice to pending in the database
+    5. We recheck the state of the invoice
+    6. We verify that the invoice is paid
+    """
+    response = await client.post(
+        "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+    assert len(invoice["payment_hash"]) == 64
+    assert len(invoice["checking_id"]) > 0
+
+    # check the payment status
+    response = await api_payment(
+        invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
+    )
+    assert response["paid"]
+
+    # make sure that the backend also thinks it's paid
+    funding_source = get_funding_source()
+    status = await funding_source.get_payment_status(invoice["payment_hash"])
+    assert status.paid
+
+    # get the outgoing payment from the db
+    payment = await get_standalone_payment(invoice["payment_hash"])
+    assert payment
+    assert payment.pending is False
+
+    # set the outgoing invoice to pending
+    await update_payment_details(payment.checking_id, pending=True)
+
+    payment_pending = await get_standalone_payment(invoice["payment_hash"])
+    assert payment_pending
+    assert payment_pending.pending is True
+
+    # check the outgoing payment status
+    await payment.check_status()
+
+    payment_not_pending = await get_standalone_payment(invoice["payment_hash"])
+    assert payment_not_pending
+    assert payment_not_pending.pending is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_hold_invoice_check_pending(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    settle_invoice(preimage)
+
+    response = await task
+    assert response.status_code < 300
+
+    # check if paid
+
+    await asyncio.sleep(1)
+
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db_after_settlement
+    assert payment_db_after_settlement.pending is False
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_hold_invoice_check_pending_and_fail(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
+
+    # cancel the hodl invoice
+    assert preimage_hash == invoice_obj.payment_hash
+    cancel_invoice(preimage_hash)
+
+    response = await task
+    assert response.status_code > 300  # should error
+
+    await asyncio.sleep(1)
+
+    # payment should not be in database anymore
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+    assert payment_db_after_settlement is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_pay_hold_invoice_check_pending_and_fail_cancel_payment_task_in_meantime(
+    client, hold_invoice, adminkey_headers_from
+):
+    preimage, invoice = hold_invoice
+    task = asyncio.create_task(
+        client.post(
+            "/api/v1/payments",
+            json={"bolt11": invoice["payment_request"]},
+            headers=adminkey_headers_from,
+        )
+    )
+    await asyncio.sleep(1)
+
+    # get payment hash from the invoice
+    invoice_obj = bolt11.decode(invoice["payment_request"])
+
+    payment_db = await get_standalone_payment(invoice_obj.payment_hash)
+
+    assert payment_db
+    assert payment_db.pending is True
+
+    # cancel payment task, this simulates the client dropping the connection
+    task.cancel()
+
+    preimage_hash = hashlib.sha256(bytes.fromhex(preimage)).hexdigest()
+
+    assert preimage_hash == invoice_obj.payment_hash
+    cancel_invoice(preimage_hash)
+
+    # check if paid
+    await asyncio.sleep(1)
+
+    # payment should still be in db
+    payment_db_after_settlement = await get_standalone_payment(invoice_obj.payment_hash)
+    assert payment_db_after_settlement is not None
+
+    # status should still be available and be False
+    status = await payment_db.check_status()
+    assert not status.paid
+
+    # now the payment should be gone after the status check
+    # payment_db_after_status_check = await get_standalone_payment(
+    #     invoice_obj.payment_hash
+    # )
+    # assert payment_db_after_status_check is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(is_fake, reason="this only works in regtest")
+async def test_receive_real_invoice_set_pending_and_check_state(
+    client, adminkey_headers_from, inkey_headers_from
+):
+    """
+    1. We create a real invoice
+    2. We pay it from our wallet
+    3. We check that the inoice was paid with the backend
+    4. We set the invoice to pending in the database
+    5. We recheck the state of the invoice with the backend
+    6. We verify that the invoice is now marked as paid in the database
+    """
+    create_invoice = CreateInvoice(out=False, amount=1000, memo="test")
+    response = await client.post(
+        "/api/v1/payments",
+        json=create_invoice.dict(),
+        headers=adminkey_headers_from,
+    )
+    assert response.status_code < 300
+    invoice = response.json()
+    response = await api_payment(
+        invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
+    )
+    assert not response["paid"]
+
+    async def listen():
+        found_checking_id = False
+        async for checking_id in get_funding_source().paid_invoices_stream():
+            if checking_id == invoice["checking_id"]:
+                found_checking_id = True
+                return
+        assert found_checking_id
+
+    task = asyncio.create_task(listen())
+    await asyncio.sleep(1)
+    pay_real_invoice(invoice["payment_request"])
+    await asyncio.wait_for(task, timeout=10)
+    response = await api_payment(
+        invoice["payment_hash"], inkey_headers_from["X-Api-Key"]
+    )
+    assert response["paid"]
+
+    # get the incoming payment from the db
+    payment = await get_standalone_payment(invoice["payment_hash"], incoming=True)
+    assert payment
+    assert payment.pending is False
+
+    # set the incoming invoice to pending
+    await update_payment_details(payment.checking_id, pending=True)
+
+    payment_pending = await get_standalone_payment(
+        invoice["payment_hash"], incoming=True
+    )
+    assert payment_pending
+    assert payment_pending.pending is True
+
+    # check the incoming payment status
+    await payment.check_status()
+
+    payment_not_pending = await get_standalone_payment(
+        invoice["payment_hash"], incoming=True
+    )
+    assert payment_not_pending
+    assert payment_not_pending.pending is False
+
+    # verify we get the same result if we use the checking_id to look up the payment
+    payment_by_checking_id = await get_standalone_payment(
+        payment_not_pending.checking_id, incoming=True
+    )
+
+    assert payment_by_checking_id
+    assert payment_by_checking_id.pending is False
+    assert payment_by_checking_id.bolt11 == payment_not_pending.bolt11
+    assert payment_by_checking_id.payment_hash == payment_not_pending.payment_hash
+
+
+@pytest.mark.asyncio
+async def test_check_fee_reserve(client, adminkey_headers_from):
+    # if regtest, create a real invoice, otherwise create an internal invoice
+    # call /api/v1/payments/fee-reserve?invoice=... with it and check if the fee reserve
+    # is correct
+    payment_request = ""
+    if is_regtest:
+        real_invoice = get_real_invoice(1000)
+        payment_request = real_invoice["payment_request"]
+
+    else:
+        create_invoice = CreateInvoice(out=False, amount=1000, memo="test")
+        response = await client.post(
+            "/api/v1/payments",
+            json=create_invoice.dict(),
+            headers=adminkey_headers_from,
+        )
+        assert response.status_code < 300
+        invoice = response.json()
+        payment_request = invoice["payment_request"]
+
+    response = await client.get(
+        f"/api/v1/payments/fee-reserve?invoice={payment_request}",
+    )
+    assert response.status_code < 300
+    fee_reserve = response.json()
+    assert fee_reserve["fee_reserve"] == fee_reserve_total(1000_000)

tests/regtest/test_x_node_api.py

@@ -0,0 +1,176 @@
+import asyncio
+import random
+from http import HTTPStatus
+
+import pytest
+from pydantic import parse_obj_as
+
+from lnbits import bolt11
+from lnbits.nodes.base import ChannelPoint, ChannelState, NodeChannel
+from tests.conftest import pytest_asyncio, settings
+
+from ..helpers import (
+    funding_source,
+    get_random_invoice_data,
+)
+from .helpers import (
+    get_unconnected_node_uri,
+    mine_blocks,
+)
+
+pytestmark = pytest.mark.skipif(
+    funding_source.__node_cls__ is None,
+    reason="Cant test if node implementation isnt available",
+)
+
+
+@pytest_asyncio.fixture()
+async def node_client(client, from_super_user):
+    settings.lnbits_node_ui = True
+    settings.lnbits_public_node_ui = False
+    settings.lnbits_node_ui_transactions = True
+    params = client.params
+    client.params = {"usr": from_super_user.id}
+    yield client
+    client.params = params
+    settings.lnbits_node_ui = False
+
+
+@pytest_asyncio.fixture()
+async def public_node_client(node_client):
+    settings.lnbits_public_node_ui = True
+    yield node_client
+    settings.lnbits_public_node_ui = False
+
+
+@pytest.mark.asyncio
+async def test_node_info_not_found(client, from_super_user):
+    settings.lnbits_node_ui = False
+    response = await client.get("/node/api/v1/info", params={"usr": from_super_user.id})
+    assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+
+
+@pytest.mark.asyncio
+async def test_public_node_info_not_found(node_client):
+    response = await node_client.get("/node/public/api/v1/info")
+    assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+
+
+@pytest.mark.asyncio
+async def test_public_node_info(public_node_client):
+    response = await public_node_client.get("/node/public/api/v1/info")
+    assert response.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_node_info(node_client):
+    response = await node_client.get("/node/api/v1/info")
+    assert response.status_code == 200
+
+
+@pytest.mark.asyncio
+async def test_node_invoices(inkey_headers_from, node_client):
+    data = await get_random_invoice_data()
+    response = await node_client.post(
+        "/api/v1/payments", json=data, headers=inkey_headers_from
+    )
+    invoice = response.json()
+
+    response = await node_client.get("/node/api/v1/invoices", params={"limit": 1})
+    assert response.status_code == 200
+    invoices = response.json()["data"]
+    assert len(invoices) == 1
+    assert invoices[0]["payment_hash"] == invoice["payment_hash"]
+
+
+@pytest.mark.asyncio
+async def test_node_payments(node_client, real_invoice, adminkey_headers_from):
+    response = await node_client.post(
+        "/api/v1/payments", json=real_invoice, headers=adminkey_headers_from
+    )
+    assert response.status_code < 300
+
+    response = await node_client.get("/node/api/v1/payments", params={"limit": 1})
+    assert response.status_code == 200
+    payments = response.json()["data"]
+    assert len(payments) == 1
+    assert (
+        payments[0]["payment_hash"]
+        == bolt11.decode(real_invoice["bolt11"]).payment_hash
+    )
+
+
+@pytest.mark.asyncio
+async def test_channel_management(node_client):
+    async def get_channels():
+        response = await node_client.get("/node/api/v1/channels")
+        assert response.status_code == 200
+        return parse_obj_as(list[NodeChannel], response.json())
+
+    data = await get_channels()
+    close = random.choice(
+        [channel for channel in data if channel.state == ChannelState.ACTIVE]
+    )
+    assert close, "No active channel found"
+    assert close.point, "No channel point found"
+
+    response = await node_client.delete(
+        "/node/api/v1/channels",
+        params={"short_id": close.short_id, **close.point.dict()},
+    )
+    assert response.status_code == 200
+
+    data = await get_channels()
+    assert any(
+        channel.point == close.point and channel.state == ChannelState.CLOSED
+        for channel in data
+    )
+
+    response = await node_client.post(
+        "/node/api/v1/channels",
+        json={
+            "peer_id": close.peer_id,
+            "funding_amount": 100000,
+        },
+    )
+    assert response.status_code == 200
+    created = ChannelPoint(**response.json())
+    data = await get_channels()
+    assert any(
+        channel.point == created and channel.state == ChannelState.PENDING
+        for channel in data
+    )
+
+    # mine some blocks so that the newly created channel eventually
+    # gets confirmed to avoid a situation where no channels are
+    # left for testing
+    mine_blocks(5)
+
+
+@pytest.mark.asyncio
+async def test_peer_management(node_client):
+    connect_uri = get_unconnected_node_uri()
+    peer_id = connect_uri.split("@")[0]
+    response = await node_client.post("/node/api/v1/peers", json={"uri": connect_uri})
+    assert response.status_code == 200
+
+    response = await node_client.get("/node/api/v1/peers")
+    assert response.status_code == 200
+    assert any(peer["id"] == peer_id for peer in response.json())
+
+    response = await node_client.delete(f"/node/api/v1/peers/{peer_id}")
+    assert response.status_code == 200
+    await asyncio.sleep(0.1)
+
+    response = await node_client.get("/node/api/v1/peers")
+    assert response.status_code == 200
+    assert not any(peer["id"] == peer_id for peer in response.json())
+
+    response = await node_client.delete(f"/node/api/v1/peers/{peer_id}")
+    assert response.status_code == 400
+
+
+@pytest.mark.asyncio
+async def test_connect_invalid_uri(node_client):
+    response = await node_client.post("/node/api/v1/peers", json={"uri": "invalid"})
+    assert response.status_code == 400