test: add boltz fundingsource to regtest (#3677)

Co-authored-by: Vlad Stan <stan.v.vlad@gmail.com>

.env.example

@@ -18,7 +18,6 @@ ENABLE_LOG_TO_FILE=true
 # https://loguru.readthedocs.io/en/stable/api/logger.html#file
 LOG_ROTATION="100 MB"
 LOG_RETENTION="3 months"
-    
 # for database cleanup commands
 # CLEANUP_WALLETS_DAYS=90
 
@@ -187,7 +186,6 @@ BOLTZ_CLIENT_ENDPOINT=127.0.0.1:9002
 BOLTZ_CLIENT_MACAROON="/home/bob/.boltz/macaroons/admin.macaroon"
 # HEXSTRING instead of path also possible
 BOLTZ_CLIENT_CERT="/home/bob/.boltz/tls.cert"
-BOLTZ_CLIENT_WALLET="lnbits"
 
 # StrikeWallet
 STRIKE_API_ENDPOINT=https://api.strike.me/v1
@@ -333,4 +331,3 @@ LNBITS_RESERVE_FEE_PERCENT=1.0
 ######################################
 ###### Logging and Development #######
 ######################################
-

.github/workflows/ci.yml

@@ -75,7 +75,14 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.10"]
-        backend-wallet-class: ["LndRestWallet", "LndWallet", "CoreLightningWallet", "CoreLightningRestWallet", "LNbitsWallet", "EclairWallet"]
+        backend-wallet-class:
+          - BoltzWallet
+          - LndRestWallet
+          - LndWallet
+          - CoreLightningWallet
+          - CoreLightningRestWallet
+          - LNbitsWallet
+          - EclairWallet
     with:
       custom-pytest: "uv run pytest tests/regtest"
       python-version: ${{ matrix.python-version }}

.github/workflows/regtest.yml

@@ -40,8 +40,8 @@ jobs:
         run: |
           git clone https://github.com/lnbits/legend-regtest-enviroment.git docker
           cd docker
-          chmod +x ./tests
+          chmod +x ./start-regtest
-          ./tests
+          ./start-regtest
           sudo chmod -R a+rwx .
 
       - name: Run pytest
@@ -63,6 +63,8 @@ jobs:
           LNBITS_ENDPOINT: http://localhost:5001
           LNBITS_KEY: "d08a3313322a4514af75d488bcc27eee"
           ECLAIR_URL: http://127.0.0.1:8082
+          BOLTZ_CLIENT_ENDPOINT: 127.0.0.1:9002
+          BOLTZ_MNEMONIC: abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon abandon about
           LNBITS_MAX_OUTGOING_PAYMENT_AMOUNT_SATS: 1000000000
           LNBITS_MAX_INCOMING_PAYMENT_AMOUNT_SATS: 1000000000
           ECLAIR_PASS: lnbits

lnbits/core/views/payment_api.py

@@ -337,7 +337,7 @@ async def api_payment(payment_hash, x_api_key: str | None = Header(None)):
         return {"paid": False, "status": "failed"}
 
     try:
-        status = await payment.check_status()
+        payment = await update_pending_payment(payment)
     except Exception:
         if wallet and wallet.id == payment.wallet_id:
             return {"paid": False, "details": payment}
@@ -346,7 +346,7 @@ async def api_payment(payment_hash, x_api_key: str | None = Header(None)):
     if wallet and wallet.id == payment.wallet_id:
         return {
             "paid": payment.success,
-            "status": f"{status!s}",
+            "status": f"{payment.status!s}",
             "preimage": payment.preimage,
             "details": payment,
         }

lnbits/wallets/boltz.py

@@ -43,12 +43,13 @@ class BoltzWallet(Wallet):
             settings.boltz_client_endpoint, add_proto=True
         )
 
-        if settings.boltz_client_macaroon:
-            self.metadata = [
-                ("macaroon", load_macaroon(settings.boltz_client_macaroon))
-            ]
-        else:
         self.metadata = None
+        if settings.boltz_client_macaroon:
+            try:
+                macaroon = load_macaroon(settings.boltz_client_macaroon)
+                self.metadata = [("macaroon", macaroon)]
+            except Exception as e:
+                logger.error(f"BoltzWallet failed to load macaroon: {e}")
 
         if settings.boltz_client_cert:
             cert = open(settings.boltz_client_cert, "rb").read()
@@ -60,17 +61,18 @@ class BoltzWallet(Wallet):
         self.rpc = boltzrpc_pb2_grpc.BoltzStub(channel)
         self.wallet_id = 0
         self.wallet_name = "lnbits"
-
+        self.wallet_ready = False
-        if settings.boltz_mnemonic:  # restore wallet from mnemonic
-            self._init_wallet_task = asyncio.create_task(
-                self._restore_boltz_wallet(
-                    settings.boltz_mnemonic, settings.boltz_client_password
-                )
-            )
-        else:  # create new wallet
-            self._init_wallet_task = asyncio.create_task(self._create_boltz_wallet())
 
     async def status(self) -> StatusResponse:
+        if self.wallet_ready is False:
+            self.wallet_ready = True
+            if settings.boltz_mnemonic:  # restore wallet from mnemonic
+                await self._restore_boltz_wallet(
+                    settings.boltz_mnemonic,
+                    settings.boltz_client_password,
+                )
+            else:  # create new wallet
+                await self._create_boltz_wallet()
         try:
             request = boltzrpc_pb2.GetWalletRequest(name=self.wallet_name)
             response: boltzrpc_pb2.Wallet = await self.rpc.GetWallet(
@@ -111,12 +113,18 @@ class BoltzWallet(Wallet):
         try:
             response = await self.rpc.CreateReverseSwap(request, metadata=self.metadata)
         except AioRpcError as exc:
+            logger.warning(exc)
             return InvoiceResponse(ok=False, error_message=exc.details())
+        fee_msat = response.routing_fee_milli_sat
         return InvoiceResponse(
-            ok=True, checking_id=response.id, payment_request=response.invoice
+            ok=True,
+            checking_id=response.id,
+            payment_request=response.invoice,
+            fee_msat=fee_msat,
         )
 
     async def pay_invoice(self, bolt11: str, fee_limit_msat: int) -> PaymentResponse:
+
         pair = boltzrpc_pb2.Pair(**{"from": boltzrpc_pb2.LBTC})
         try:
             pair_info: boltzrpc_pb2.PairInfo
@@ -128,8 +136,9 @@ class BoltzWallet(Wallet):
 
             if not invoice.amount_msat:
                 raise ValueError("amountless invoice")
+
             service_fee: float = invoice.amount_msat * pair_info.fees.percentage / 100
-            estimate = service_fee + pair_info.fees.miner_fees * 1000
+            estimate = int(service_fee + pair_info.fees.miner_fees * 1000)
             if estimate > fee_limit_msat:
                 error = f"fee of {estimate} msat exceeds limit of {fee_limit_msat} msat"
 
@@ -151,11 +160,12 @@ class BoltzWallet(Wallet):
             if response.id == "":
                 # note that there is no way to provide a checking id here,
                 # but there is no need since it immediately is considered as successfull
-                return PaymentResponse(
+                logger.warning(
-                    ok=True,
+                    "Boltz invoice paid directly on liquid network using magic routing"
-                    checking_id=response.id,
                 )
+                return PaymentResponse(ok=True, checking_id=invoice.payment_hash)
         except AioRpcError as exc:
+            logger.warning(exc)
             return PaymentResponse(ok=False, error_message=exc.details())
 
         try:
@@ -165,10 +175,15 @@ class BoltzWallet(Wallet):
                 info_request, metadata=self.metadata
             ):
                 if info.swap.state == boltzrpc_pb2.SUCCESSFUL:
+                    fee_msat = (info.swap.onchain_fee + info.swap.service_fee) * 1000
+                    logger.debug(
+                        f"Boltz swap successful, status: {info.swap.status}"
+                        f"fee_msat: {fee_msat}"
+                    )
                     return PaymentResponse(
                         ok=True,
-                        checking_id=response.id,
+                        checking_id=invoice.payment_hash,
-                        fee_msat=(info.swap.onchain_fee + info.swap.service_fee) * 1000,
+                        fee_msat=fee_msat,
                         preimage=info.swap.preimage,
                     )
                 elif info.swap.error != "":
@@ -177,21 +192,29 @@ class BoltzWallet(Wallet):
                 ok=False, error_message="stream stopped unexpectedly"
             )
         except AioRpcError as exc:
+            logger.warning(exc)
             return PaymentResponse(ok=False, error_message=exc.details())
 
     async def get_invoice_status(self, checking_id: str) -> PaymentStatus:
         try:
+            request = boltzrpc_pb2.GetSwapInfoRequest(id=checking_id)
             response: boltzrpc_pb2.GetSwapInfoResponse = await self.rpc.GetSwapInfo(
-                boltzrpc_pb2.GetSwapInfoRequest(id=checking_id), metadata=self.metadata
+                request,
+                metadata=self.metadata,
             )
             swap = response.reverse_swap
-        except AioRpcError:
+        except AioRpcError as exc:
+            logger.warning(exc)
             return PaymentPendingStatus()
         if swap.state == boltzrpc_pb2.SwapState.SUCCESSFUL:
+            fee_msat = (
+                swap.service_fee + swap.onchain_fee
+            ) * 1000 + swap.routing_fee_msat
+            logger.debug(
+                f"Boltz swap successful, status: {swap.status}, fee_msat: {fee_msat}"
+            )
             return PaymentSuccessStatus(
-                fee_msat=(
+                fee_msat=fee_msat,
-                    (swap.service_fee + swap.onchain_fee) * 1000 + swap.routing_fee_msat
-                ),
                 preimage=swap.preimage,
             )
         elif swap.state == boltzrpc_pb2.SwapState.PENDING:
@@ -201,18 +224,23 @@ class BoltzWallet(Wallet):
 
     async def get_payment_status(self, checking_id: str) -> PaymentStatus:
         try:
+            checking_id_bytes = bytes.fromhex(checking_id)
+            request = boltzrpc_pb2.GetSwapInfoRequest(payment_hash=checking_id_bytes)
             response: boltzrpc_pb2.GetSwapInfoResponse = await self.rpc.GetSwapInfo(
-                boltzrpc_pb2.GetSwapInfoRequest(
+                request,
-                    payment_hash=bytes.fromhex(checking_id)
-                ),
                 metadata=self.metadata,
             )
             swap = response.swap
-        except AioRpcError:
+        except AioRpcError as exc:
+            logger.warning(exc)
             return PaymentPendingStatus()
         if swap.state == boltzrpc_pb2.SwapState.SUCCESSFUL:
+            fee_msat = (swap.service_fee + swap.onchain_fee) * 1000
+            logger.debug(
+                f"Boltz swap successful, status: {swap.status}, fee_msat: {fee_msat}"
+            )
             return PaymentSuccessStatus(
-                fee_msat=(swap.service_fee + swap.onchain_fee) * 1000,
+                fee_msat=fee_msat,
                 preimage=swap.preimage,
             )
         elif swap.state == boltzrpc_pb2.SwapState.PENDING:
@@ -229,7 +257,16 @@ class BoltzWallet(Wallet):
                     request, metadata=self.metadata
                 ):
                     reverse = info.reverse_swap
-                    if reverse and reverse.state == boltzrpc_pb2.SUCCESSFUL:
+                    if (
+                        reverse
+                        and reverse.state == boltzrpc_pb2.SUCCESSFUL
+                        and reverse.status == "invoice.settled"
+                    ):
+                        fee_msat = ((reverse.service_fee + reverse.onchain_fee) * 1000,)
+                        logger.debug(
+                            f"Boltz reverse swap settled: {reverse.id}, "
+                            f"fee_msat: {fee_msat}"
+                        )
                         yield reverse.id
             except Exception as exc:
                 logger.error(
@@ -244,13 +281,8 @@ class BoltzWallet(Wallet):
         try:
             request = boltzrpc_pb2.GetWalletRequest(name=wallet_name)
             response = await self.rpc.GetWallet(request, metadata=self.metadata)
-            logger.info(f"Wallet '{wallet_name}' already exists with ID {response.id}")
             return response
-        except AioRpcError as exc:
+        except AioRpcError:
-            if exc.code() != grpc.StatusCode.NOT_FOUND:
-                logger.warning(f"Wallet '{wallet_name}' does not exist.")
-                return None
-            logger.error(f"Error checking wallet existence: {exc.details()}")
             return None
 
     async def _delete_wallet(self, wallet_id: int) -> None:

tests/conftest.py

@@ -331,8 +331,8 @@ def _settings_cleanup(settings: Settings):
     settings.lnbits_allowed_users = []
     settings.auth_allowed_methods = AuthMethods.all()
     settings.auth_credetials_update_threshold = 120
-    settings.lnbits_reserve_fee_percent = 1
+    settings.lnbits_reserve_fee_percent = 2
-    settings.lnbits_reserve_fee_min = 2000
+    settings.lnbits_reserve_fee_min = 20000
     settings.lnbits_service_fee = 0
     settings.lnbits_reserve_fee_percent = 0
     settings.lnbits_wallet_limit_daily_max_withdraw = 0

tests/regtest/helpers.py

@@ -6,6 +6,11 @@ from subprocess import PIPE, Popen, TimeoutExpired
 
 from loguru import logger
 
+from lnbits.wallets import get_funding_source
+
+funding_source = get_funding_source()
+is_boltz_wallet = funding_source.__class__.__name__ == "BoltzWallet"
+
 docker_lightning_cli = [
     "docker",
     "exec",
@@ -27,6 +32,16 @@ docker_bitcoin_cli = [
 ]
 
 
+docker_elements_cli = [
+    "docker",
+    "exec",
+    "lnbits-elementsd-1",
+    "elements-cli",
+    "-rpcport=18884",
+    "-chain=liquidregtest",
+]
+
+
 docker_lightning_unconnected_cli = [
     "docker",
     "exec",
@@ -50,7 +65,7 @@ docker_lightning_noroute_cli = [
 
 
 def run_cmd(cmd: list) -> str:
-    timeout = 10
+    timeout = 30
     process = Popen(cmd, stdout=PIPE, stderr=PIPE)
 
     logger.debug(f"running command: {cmd}")
@@ -128,6 +143,12 @@ def mine_blocks(blocks: int = 1) -> str:
     return run_cmd(cmd)
 
 
+def mine_blocks_liquid(blocks: int = 1) -> str:
+    cmd = docker_elements_cli.copy()
+    cmd.extend(["-generate", str(blocks)])
+    return run_cmd(cmd)
+
+
 def get_unconnected_node_uri() -> str:
     cmd = docker_lightning_unconnected_cli.copy()
     cmd.append("getinfo")

tests/regtest/test_real_invoice.py

@@ -18,6 +18,7 @@ from ..helpers import FakeError, is_fake, is_regtest
 from .helpers import (
     cancel_invoice,
     get_real_invoice,
+    mine_blocks_liquid,
     pay_real_invoice,
     settle_invoice,
 )
@@ -66,7 +67,7 @@ async def test_pay_real_invoice(
 
     await asyncio.sleep(1)
     balance = await get_node_balance_sats()
-    assert prev_balance - balance == 100
+    assert prev_balance - balance == 100 + abs(payment.fee // 1000)
 
 
 @pytest.mark.anyio
@@ -153,7 +154,8 @@ async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_
 
     async def on_paid(payment: Payment):
 
-        assert payment.checking_id == invoice["payment_hash"]
+        assert payment.payment_hash == invoice["payment_hash"]
+        assert payment.checking_id == invoice["checking_id"]
 
         response = await client.get(
             f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
@@ -164,7 +166,8 @@ async def test_create_real_invoice(client, adminkey_headers_from, inkey_headers_
 
         await asyncio.sleep(1)
         balance = await get_node_balance_sats()
-        assert balance - prev_balance == create_invoice.amount
+        fee = abs(payment_status.get("details", {}).get("fee", 0) // 1000)
+        assert balance - prev_balance == create_invoice.amount - fee
 
         assert payment_status.get("preimage") is not None
 
@@ -200,6 +203,8 @@ async def test_pay_real_invoice_set_pending_and_check_state(
     assert len(invoice["payment_hash"]) == 64
     assert len(invoice["checking_id"]) > 0
 
+    mine_blocks_liquid(1)
+    await asyncio.sleep(1)
     # check the payment status
     response = await client.get(
         f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from
@@ -341,6 +346,7 @@ async def test_pay_hold_invoice_check_pending_and_fail_cancel_payment_task_in_me
     assert preimage_hash == invoice_obj.payment_hash
     cancel_invoice(preimage_hash)
 
+    mine_blocks_liquid(1)
     # check if paid
     await asyncio.sleep(1)
 
@@ -382,7 +388,9 @@ async def test_receive_real_invoice_set_pending_and_check_state(
     assert not payment_status["paid"]
 
     async def on_paid(payment: Payment):
-        assert payment.checking_id == invoice["payment_hash"]
+
+        assert payment.payment_hash == invoice["payment_hash"]
+        assert payment.checking_id == invoice["checking_id"]
 
         response = await client.get(
             f'/api/v1/payments/{invoice["payment_hash"]}', headers=inkey_headers_from

tests/regtest/test_services_create_invoice.py

@@ -7,6 +7,8 @@ from lnbits.core.services import (
 from lnbits.wallets import get_funding_source
 from lnbits.wallets.base import PaymentStatus
 
+from .helpers import is_boltz_wallet
+
 description = "test create invoice"
 
 
@@ -17,6 +19,9 @@ async def test_create_invoice(from_wallet):
         amount=1000,
         memo=description,
     )
+
+    # we cannot know the preimage of the swap yet
+    if not is_boltz_wallet:
         assert payment.preimage
 
     invoice = decode(payment.bolt11)
@@ -35,6 +40,9 @@ async def test_create_internal_invoice(from_wallet):
     payment = await create_invoice(
         wallet_id=from_wallet.id, amount=1000, memo=description, internal=True
     )
+
+    # we cannot know the preimage of the swap yet
+    if not is_boltz_wallet:
         assert payment.preimage
 
     invoice = decode(payment.bolt11)

tests/regtest/test_services_pay_invoice.py

@@ -6,6 +6,8 @@ from lnbits.core.services import (
 )
 from lnbits.exceptions import PaymentError
 
+from .helpers import is_boltz_wallet
+
 description = "test pay invoice"
 
 
@@ -17,9 +19,12 @@ async def test_services_pay_invoice(to_wallet, real_invoice):
         description=description,
     )
     assert payment
-    assert payment.status == PaymentState.SUCCESS
     assert payment.memo == description
+    if not is_boltz_wallet:
+        assert payment.status == PaymentState.SUCCESS
         assert payment.preimage
+    else:
+        assert payment.status == PaymentState.PENDING
 
 
 @pytest.mark.anyio