Merge pull request #1033 from lnbits/startup/skip_invoice_expriy_check

On-the-fly invoice expiry check

lnbits/core/crud.py

@@ -339,36 +339,13 @@ async def delete_expired_invoices(
           AND time < {db.timestamp_now} - {db.interval_seconds(2592000)}
         """
     )
-
+    # then we delete all invoices whose expiry date is in the past
-    # then we delete all expired invoices, checking one by one
-    rows = await (conn or db).fetchall(
-        f"""
-        SELECT bolt11
-        FROM apipayments
-        WHERE pending = true
-          AND bolt11 IS NOT NULL
-          AND amount > 0 AND time < {db.timestamp_now} - {db.interval_seconds(86400)}
-        """
-    )
-    logger.debug(f"Checking expiry of {len(rows)} invoices")
-    for i, (payment_request,) in enumerate(rows):
-        try:
-            invoice = bolt11.decode(payment_request)
-        except:
-            continue
-
-        expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry)
-        if expiration_date > datetime.datetime.utcnow():
-            continue
-        logger.debug(
-            f"Deleting expired invoice {i}/{len(rows)}: {invoice.payment_hash} (expired: {expiration_date})"
-        )
     await (conn or db).execute(
-            """
+        f"""
         DELETE FROM apipayments
-            WHERE pending = true AND hash = ?
+        WHERE pending = true AND amount > 0
-            """,
+          AND expiry < {db.timestamp_now}
-            (invoice.payment_hash,),
+        """
     )
 
 
@@ -396,12 +373,19 @@ async def create_payment(
     # previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn)
     # assert previous_payment is None, "Payment already exists"
 
+    try:
+        invoice = bolt11.decode(payment_request)
+        expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry)
+    except:
+        # assume maximum bolt11 expiry of 31 days to be on the safe side
+        expiration_date = datetime.datetime.now() + datetime.timedelta(days=31)
+
     await (conn or db).execute(
         """
         INSERT INTO apipayments
           (wallet, checking_id, bolt11, hash, preimage,
-           amount, pending, memo, fee, extra, webhook)
+           amount, pending, memo, fee, extra, webhook, expiry)
-        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
         """,
         (
             wallet_id,
@@ -417,6 +401,7 @@ async def create_payment(
             if extra and extra != {} and type(extra) is dict
             else None,
             webhook,
+            db.datetime_to_timestamp(expiration_date),
         ),
     )
 

lnbits/core/migrations.py

@@ -1,5 +1,10 @@
+import datetime
+
+from loguru import logger
 from sqlalchemy.exc import OperationalError  # type: ignore
 
+from lnbits import bolt11
+
 
 async def m000_create_migrations_table(db):
     await db.execute(
@@ -188,3 +193,68 @@ async def m005_balance_check_balance_notify(db):
         );
     """
     )
+
+
+async def m006_add_invoice_expiry_to_apipayments(db):
+    """
+    Adds invoice expiry column to apipayments.
+    """
+    try:
+        await db.execute("ALTER TABLE apipayments ADD COLUMN expiry TIMESTAMP")
+    except OperationalError:
+        pass
+
+
+async def m007_set_invoice_expiries(db):
+    """
+    Precomputes invoice expiry for existing pending incoming payments.
+    """
+    try:
+        rows = await (
+            await db.execute(
+                f"""
+                SELECT bolt11, checking_id
+                FROM apipayments
+                WHERE pending = true
+                AND amount > 0
+                AND bolt11 IS NOT NULL
+                AND expiry IS NULL
+                AND time < {db.timestamp_now}
+                """
+            )
+        ).fetchall()
+        if len(rows):
+            logger.info(f"Mirgraion: Checking expiry of {len(rows)} invoices")
+        for i, (
+            payment_request,
+            checking_id,
+        ) in enumerate(rows):
+            try:
+                invoice = bolt11.decode(payment_request)
+                if invoice.expiry is None:
+                    continue
+
+                expiration_date = datetime.datetime.fromtimestamp(
+                    invoice.date + invoice.expiry
+                )
+                logger.info(
+                    f"Mirgraion: {i+1}/{len(rows)} setting expiry of invoice {invoice.payment_hash} to {expiration_date}"
+                )
+                await db.execute(
+                    """
+                    UPDATE apipayments SET expiry = ?
+                    WHERE checking_id = ? AND amount > 0
+                    """,
+                    (
+                        db.datetime_to_timestamp(expiration_date),
+                        checking_id,
+                    ),
+                )
+            except:
+                continue
+    except OperationalError:
+        # this is necessary now because it may be the case that this migration will
+        # run twice in some environments.
+        # catching errors like this won't be necessary in anymore now that we
+        # keep track of db versions so no migration ever runs twice.
+        pass

lnbits/core/models.py

@@ -1,6 +1,8 @@
+import datetime
 import hashlib
 import hmac
 import json
+import time
 from sqlite3 import Row
 from typing import Dict, List, NamedTuple, Optional
 
@@ -83,6 +85,7 @@ class Payment(BaseModel):
     bolt11: str
     preimage: str
     payment_hash: str
+    expiry: Optional[float]
     extra: Optional[Dict] = {}
     wallet_id: str
     webhook: Optional[str]
@@ -101,6 +104,7 @@ class Payment(BaseModel):
             fee=row["fee"],
             memo=row["memo"],
             time=row["time"],
+            expiry=row["expiry"],
             wallet_id=row["wallet"],
             webhook=row["webhook"],
             webhook_status=row["webhook_status"],
@@ -128,6 +132,10 @@ class Payment(BaseModel):
     def is_out(self) -> bool:
         return self.amount < 0
 
+    @property
+    def is_expired(self) -> bool:
+        return self.expiry < time.time() if self.expiry else False
+
     @property
     def is_uncheckable(self) -> bool:
         return self.checking_id.startswith("internal_")
@@ -170,7 +178,13 @@ class Payment(BaseModel):
 
         logger.debug(f"Status: {status}")
 
-        if self.is_out and status.failed:
+        if self.is_in and status.pending and self.is_expired and self.expiry:
+            expiration_date = datetime.datetime.fromtimestamp(self.expiry)
+            logger.debug(
+                f"Deleting expired incoming pending payment {self.checking_id}: expired {expiration_date}"
+            )
+            await self.delete(conn)
+        elif self.is_out and status.failed:
             logger.warning(
                 f"Deleting outgoing failed payment {self.checking_id}: {status}"
             )

lnbits/db.py

@@ -29,6 +29,13 @@ class Compat:
             return f"{seconds}"
         return "<nothing>"
 
+    def datetime_to_timestamp(self, date: datetime.datetime):
+        if self.type in {POSTGRES, COCKROACH}:
+            return date.strftime("%Y-%m-%d %H:%M:%S")
+        elif self.type == SQLITE:
+            return time.mktime(date.timetuple())
+        return "<nothing>"
+
     @property
     def timestamp_now(self) -> str:
         if self.type in {POSTGRES, COCKROACH}:

lnbits/static/js/base.js

@@ -184,6 +184,7 @@ window.LNbits = {
         bolt11: data.bolt11,
         preimage: data.preimage,
         payment_hash: data.payment_hash,
+        expiry: data.expiry,
         extra: data.extra,
         wallet_id: data.wallet_id,
         webhook: data.webhook,
@@ -195,6 +196,11 @@ window.LNbits = {
         'YYYY-MM-DD HH:mm'
       )
       obj.dateFrom = moment(obj.date).fromNow()
+      obj.expirydate = Quasar.utils.date.formatDate(
+        new Date(obj.expiry * 1000),
+        'YYYY-MM-DD HH:mm'
+      )
+      obj.expirydateFrom = moment(obj.expirydate).fromNow()
       obj.msat = obj.amount
       obj.sat = obj.msat / 1000
       obj.tag = obj.extra.tag

lnbits/static/js/components.js

@@ -192,9 +192,13 @@ Vue.component('lnbits-payment-details', {
         </q-badge>
       </div>
       <div class="row">
-        <div class="col-3"><b>Date</b>:</div>
+        <div class="col-3"><b>Created</b>:</div>
         <div class="col-9">{{ payment.date }} ({{ payment.dateFrom }})</div>
       </div>
+      <div class="row">
+        <div class="col-3"><b>Expiry</b>:</div>
+        <div class="col-9">{{ payment.expirydate }} ({{ payment.expirydateFrom }})</div>
+      </div>      
       <div class="row">
         <div class="col-3"><b>Description</b>:</div>
         <div class="col-9">{{ payment.memo }}</div>

lnbits/tasks.py

@@ -145,7 +145,7 @@ async def check_pending_payments():
             )
             # we delete expired invoices once upon the first pending check
             if incoming:
-                logger.info("Task: deleting all expired invoices")
+                logger.debug("Task: deleting all expired invoices")
                 start_time: float = time.time()
                 await delete_expired_invoices(conn=conn)
                 logger.info(