diff --git a/lnbits/core/crud.py b/lnbits/core/crud.py index cbed6292..ae8de7ab 100644 --- a/lnbits/core/crud.py +++ b/lnbits/core/crud.py @@ -321,37 +321,45 @@ async def delete_expired_invoices( AND time < {db.timestamp_now} - {db.interval_seconds(2592000)} """ ) - - # then we delete all expired invoices, checking one by one - rows = await (conn or db).fetchall( + # then we delete all invoices whose expiry date is in the past + await (conn or db).execute( f""" - SELECT bolt11 - FROM apipayments - WHERE pending = true - AND bolt11 IS NOT NULL - AND amount > 0 AND time < {db.timestamp_now} - {db.interval_seconds(86400)} + DELETE FROM apipayments + WHERE pending = true AND amount > 0 + AND expiry < {db.timestamp_now} """ ) - logger.debug(f"Checking expiry of {len(rows)} invoices") - for (payment_request,) in rows: - try: - invoice = bolt11.decode(payment_request) - except: - continue - expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry) - if expiration_date > datetime.datetime.utcnow(): - continue - logger.debug( - f"Deleting expired invoice: {invoice.payment_hash} (expired: {expiration_date})" - ) - await (conn or db).execute( - """ - DELETE FROM apipayments - WHERE pending = true AND hash = ? - """, - (invoice.payment_hash,), - ) + # # then we delete all expired invoices, checking one by one + # rows = await (conn or db).fetchall( + # f""" + # SELECT bolt11 + # FROM apipayments + # WHERE pending = true + # AND bolt11 IS NOT NULL + # AND amount > 0 AND time < {db.timestamp_now} - {db.interval_seconds(86400)} + # """ + # ) + # logger.debug(f"Checking expiry of {len(rows)} invoices") + # for (payment_request,) in rows: + # try: + # invoice = bolt11.decode(payment_request) + # except: + # continue + + # expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry) + # if expiration_date > datetime.datetime.utcnow(): + # continue + # logger.debug( + # f"Deleting expired invoice: {invoice.payment_hash} (expired: {expiration_date})" + # ) + # await (conn or db).execute( + # """ + # DELETE FROM apipayments + # WHERE pending = true AND hash = ? + # """, + # (invoice.payment_hash,), + # ) # payments @@ -375,15 +383,21 @@ async def create_payment( ) -> Payment: # todo: add this when tests are fixed - # previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn) - # assert previous_payment is None, "Payment already exists" + previous_payment = await get_wallet_payment(wallet_id, payment_hash, conn=conn) + assert previous_payment is None, "Payment already exists" + + try: + invoice = bolt11.decode(payment_request) + expiration_date = datetime.datetime.fromtimestamp(invoice.date + invoice.expiry) + except: + expiration_date = datetime.datetime.now() + datetime.timedelta(days=31) await (conn or db).execute( """ INSERT INTO apipayments (wallet, checking_id, bolt11, hash, preimage, - amount, pending, memo, fee, extra, webhook) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + amount, pending, memo, fee, extra, webhook, expiry) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( wallet_id, @@ -399,6 +413,7 @@ async def create_payment( if extra and extra != {} and type(extra) is dict else None, webhook, + expiration_date, ), ) diff --git a/lnbits/core/migrations.py b/lnbits/core/migrations.py index ebecb5e3..e422df08 100644 --- a/lnbits/core/migrations.py +++ b/lnbits/core/migrations.py @@ -1,4 +1,6 @@ from sqlalchemy.exc import OperationalError # type: ignore +from lnbits import bolt11 +import datetime async def m000_create_migrations_table(db): @@ -188,3 +190,60 @@ async def m005_balance_check_balance_notify(db): ); """ ) + + +async def m006_add_invoice_expiry_to_apipayments(db): + """ + Adds invoice expiry field to apipayments and precomputes them for + existing entries + """ + try: + rows = await ( + await db.execute( + f""" + SELECT bolt11, checking_id + FROM apipayments + WHERE pending = true + AND bolt11 IS NOT NULL + AND expiry IS NULL + AND amount > 0 AND time < {db.timestamp_now} - {db.interval_seconds(86400)} + """ + ) + ).fetchall() + # then we delete all expired invoices, checking one by one + print(f"Checking expiry of {len(rows)} invoices") + for i, ( + payment_request, + payment_hash, + ) in enumerate(rows): + print(f"Checking invoice {i}/{len(rows)}") + try: + invoice = bolt11.decode(payment_request) + except: + continue + if payment_hash != invoice.payment_hash: + print("Error: {payment_hash} != {invoice.payment_hash}") + continue + + expiration_date = datetime.datetime.fromtimestamp( + invoice.date + invoice.expiry + ) + print( + f"Setting expiry of invoice {invoice.payment_hash} to {expiration_date}" + ) + await db.execute( + """ + UPDATE apipayments SET expiry = ? + WHERE checking_id = ? AND amount > 0 + """, + ( + expiration_date, + invoice.payment_hash, + ), + ) + except OperationalError: + # this is necessary now because it may be the case that this migration will + # run twice in some environments. + # catching errors like this won't be necessary in anymore now that we + # keep track of db versions so no migration ever runs twice. + pass diff --git a/poetry.lock b/poetry.lock index 343fffbf..281b9474 100644 --- a/poetry.lock +++ b/poetry.lock @@ -659,12 +659,12 @@ optional = false python-versions = ">=3.7,<4.0" [package.dependencies] -pyln-bolt7 = ">=1.0" -pyln-proto = ">=0.12" +pyln-bolt7 = ">=1.0,<2.0" +pyln-proto = ">=0.11,<0.12" [[package]] name = "pyln-proto" -version = "0.12.0" +version = "0.11.1" description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)." category = "main" optional = false @@ -839,14 +839,14 @@ cffi = ">=1.3.0" [[package]] name = "setuptools" -version = "65.4.0" +version = "65.4.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "main" optional = false python-versions = ">=3.7" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] @@ -1051,7 +1051,7 @@ testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black ( [metadata] lock-version = "1.1" python-versions = "^3.10 | ^3.9 | ^3.8 | ^3.7" -content-hash = "72e4462285d0bc5e2cb83c88c613726beced959b268bd30b984d8baaeff178ea" +content-hash = "1a0341703a38328d0b00ab169a02b238d88b2d51ab8663da5591d3ee32de55ec" [metadata.files] aiofiles = [ @@ -1696,8 +1696,8 @@ pyln-client = [ {file = "pyln_client-0.11.1-py3-none-any.whl", hash = "sha256:497db443406b80c98c0434e2938eb1b2a17e88fd9aa63b018124068198df6141"}, ] pyln-proto = [ - {file = "pyln-proto-0.12.0.tar.gz", hash = "sha256:3214d99d8385f2135a94937f0dc1da626a33b257e9ebc320841656edaefabbe5"}, - {file = "pyln_proto-0.12.0-py3-none-any.whl", hash = "sha256:dedef5d8e476a9ade5a0b2eb919ccc37e4a57f2a78fdc399f1c5e0de17e41604"}, + {file = "pyln-proto-0.11.1.tar.gz", hash = "sha256:9bed240f41917c4fd526b767218a77d0fbe69242876eef72c35a856796f922d6"}, + {file = "pyln_proto-0.11.1-py3-none-any.whl", hash = "sha256:27b2e04a81b894f69018279c0ce4aa2e7ccd03b86dd9783f96b9d8d1498c8393"}, ] pyparsing = [ {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, @@ -1799,8 +1799,8 @@ secp256k1 = [ {file = "secp256k1-0.14.0.tar.gz", hash = "sha256:82c06712d69ef945220c8b53c1a0d424c2ff6a1f64aee609030df79ad8383397"}, ] setuptools = [ - {file = "setuptools-65.4.0-py3-none-any.whl", hash = "sha256:c2d2709550f15aab6c9110196ea312f468f41cd546bceb24127a1be6fdcaeeb1"}, - {file = "setuptools-65.4.0.tar.gz", hash = "sha256:a8f6e213b4b0661f590ccf40de95d28a177cd747d098624ad3f69c40287297e9"}, + {file = "setuptools-65.4.1-py3-none-any.whl", hash = "sha256:1b6bdc6161661409c5f21508763dc63ab20a9ac2f8ba20029aaaa7fdb9118012"}, + {file = "setuptools-65.4.1.tar.gz", hash = "sha256:3050e338e5871e70c72983072fe34f6032ae1cdeeeb67338199c2f74e083a80e"}, ] shortuuid = [ {file = "shortuuid-1.0.1-py3-none-any.whl", hash = "sha256:492c7402ff91beb1342a5898bd61ea953985bf24a41cd9f247409aa2e03c8f77"}, diff --git a/pyproject.toml b/pyproject.toml index 19dac860..40bf3c04 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ protobuf = "^4.21.6" Cerberus = "^1.3.4" async-timeout = "^4.0.2" pyln-client = "0.11.1" +setuptools = "^65.4.1" [tool.poetry.dev-dependencies] isort = "^5.10.1"