Merge branch 'main' into diagon-alley

.github/workflows/migrations.yml

@@ -9,9 +9,9 @@ jobs:
       postgres:
         image: postgres:latest
         env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
+          POSTGRES_USER: lnbits
+          POSTGRES_PASSWORD: lnbits
+          POSTGRES_DB: migration
         ports:
           # maps tcp port 5432 on service container to the host
           - 5432:5432
@@ -36,11 +36,4 @@ jobs:
           sudo apt install unzip
       - name: Run migrations
         run: |
-          rm -rf ./data
-          mkdir -p ./data
-          export LNBITS_DATA_FOLDER="./data"
-          unzip tests/data/mock_data.zip -d ./data
-          timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
-          export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres"
-          timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
-          poetry run python tools/conv.py
+          make test-migration

Makefile

@@ -4,7 +4,7 @@ all: format check requirements.txt
 
 format: prettier isort black
 
-check: mypy checkprettier checkisort checkblack 
+check: mypy checkprettier checkisort checkblack
 
 prettier: $(shell find lnbits -name "*.js" -name ".html")
 	./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js lnbits/extensions/*/static/components/*/*.js  lnbits/extensions/*/static/components/*/*.html
@@ -32,11 +32,13 @@ test:
 	FAKE_WALLET_SECRET="ToTheMoon1" \
 	LNBITS_DATA_FOLDER="./tests/data" \
 	PYTHONUNBUFFERED=1 \
+	DEBUG=true \
 	poetry run pytest
 
 test-real-wallet:
 	LNBITS_DATA_FOLDER="./tests/data" \
 	PYTHONUNBUFFERED=1 \
+	DEBUG=true \
 	poetry run pytest
 
 test-venv:
@@ -44,7 +46,27 @@ test-venv:
 	FAKE_WALLET_SECRET="ToTheMoon1" \
 	LNBITS_DATA_FOLDER="./tests/data" \
 	PYTHONUNBUFFERED=1 \
+	DEBUG=true \
 	./venv/bin/pytest --durations=1 -s --cov=lnbits --cov-report=xml tests
 
+test-migration:
+	rm -rf ./migration-data
+	mkdir -p ./migration-data
+	unzip tests/data/mock_data.zip -d ./migration-data
+	HOST=0.0.0.0 \
+	PORT=5002 \
+	LNBITS_DATA_FOLDER="./migration-data" \
+	timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+	HOST=0.0.0.0 \
+	PORT=5002 \
+	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \
+	timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi
+	LNBITS_DATA_FOLDER="./migration-data" \
+	LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \
+	poetry run python tools/conv.py
+
+migration:
+	poetry run python tools/conv.py
+
 bak:
 	# LNBITS_DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432/postgres

docs/devs/extensions.md

@@ -48,4 +48,25 @@ LNbits currently supports SQLite and PostgreSQL databases. There is a migration
 
 ### Adding mock data to `mock_data.zip`
 
-`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR.
+`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3`, updated `database.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR.
+
+### running migration locally
+you will need a running postgres database
+
+#### create lnbits user for migration database
+```console
+sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'"
+```
+#### create migration database
+```console
+sudo su - postgres -c "psql -c 'CREATE DATABASE migration;'"
+```
+#### run the migration
+```console
+make test-migration
+```
+sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'"
+#### clean migration database afterwards, fails if you try again
+```console
+sudo su - postgres -c "psql -c 'DROP DATABASE IF EXISTS migration;'"
+```

docs/guide/installation.md

@@ -170,8 +170,9 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits"
 
 # START LNbits
 # STOP LNbits
-# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials
-python3 tools/conv.py
+poetry run python tools/conv.py
+# or
+make migration
 ```
 
 Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly.
@@ -194,15 +195,14 @@ Description=LNbits
 
 [Service]
 # replace with the absolute path of your lnbits installation
-WorkingDirectory=/home/bitcoin/lnbits
-# same here
-ExecStart=/home/bitcoin/lnbits/venv/bin/uvicorn lnbits.__main__:app --port 5000
+WorkingDirectory=/home/lnbits/lnbits-legend
+# same here. run `which poetry` if you can't find the poetry binary
+ExecStart=/home/lnbits/.local/bin/poetry run lnbits
 # replace with the user that you're running lnbits on
-User=bitcoin
+User=lnbits
 Restart=always
 TimeoutSec=120
 RestartSec=30
-# this makes sure that you receive logs in real time
 Environment=PYTHONUNBUFFERED=1
 
 [Install]

lnbits/core/static/js/wallet.js

@@ -668,7 +668,17 @@ new Vue({
       })
     },
     exportCSV: function () {
-      LNbits.utils.exportCSV(this.paymentsTable.columns, this.payments)
+      // status is important for export but it is not in paymentsTable
+      // because it is manually added with payment detail link and icons
+      // and would cause duplication in the list
+      let columns = this.paymentsTable.columns
+      columns.unshift({
+        name: 'pending',
+        align: 'left',
+        label: 'Pending',
+        field: 'pending'
+      })
+      LNbits.utils.exportCSV(columns, this.payments)
     }
   },
   watch: {

lnbits/core/views/api.py

@@ -1,7 +1,7 @@
 import asyncio
+import binascii
 import hashlib
 import json
-from binascii import unhexlify
 from http import HTTPStatus
 from io import BytesIO
 from typing import Dict, List, Optional, Tuple, Union
@@ -152,11 +152,23 @@ class CreateInvoiceData(BaseModel):
 
 async def api_payments_create_invoice(data: CreateInvoiceData, wallet: Wallet):
     if data.description_hash:
-        description_hash = unhexlify(data.description_hash)
+        try:
+            description_hash = binascii.unhexlify(data.description_hash)
+        except binascii.Error:
+            raise HTTPException(
+                status_code=HTTPStatus.BAD_REQUEST,
+                detail="'description_hash' must be a valid hex string",
+            )
         unhashed_description = b""
         memo = ""
     elif data.unhashed_description:
-        unhashed_description = unhexlify(data.unhashed_description)
+        try:
+            unhashed_description = binascii.unhexlify(data.unhashed_description)
+        except binascii.Error:
+            raise HTTPException(
+                status_code=HTTPStatus.BAD_REQUEST,
+                detail="'unhashed_description' must be a valid hex string",
+            )
         description_hash = b""
         memo = ""
     else:

lnbits/decorators.py

@@ -130,10 +130,13 @@ async def get_key_type(
     # 2: invalid
     pathname = r["path"].split("/")[1]
 
-    if not api_key_header and not api_key_query:
-        raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST)
+    token = api_key_header or api_key_query
 
-    token = api_key_header if api_key_header else api_key_query
+    if not token:
+        raise HTTPException(
+            status_code=HTTPStatus.UNAUTHORIZED,
+            detail="Invoice (or Admin) key required.",
+        )
 
     try:
         admin_checker = WalletAdminKeyChecker(api_key=token)
@@ -180,7 +183,14 @@ async def require_admin_key(
     api_key_header: str = Security(api_key_header),  # type: ignore
     api_key_query: str = Security(api_key_query),  # type: ignore
 ):
-    token = api_key_header if api_key_header else api_key_query
+
+    token = api_key_header or api_key_query
+
+    if not token:
+        raise HTTPException(
+            status_code=HTTPStatus.UNAUTHORIZED,
+            detail="Admin key required.",
+        )
 
     wallet = await get_key_type(r, token)
 
@@ -199,11 +209,12 @@ async def require_invoice_key(
     api_key_header: str = Security(api_key_header),  # type: ignore
     api_key_query: str = Security(api_key_query),  # type: ignore
 ):
+
     token = api_key_header or api_key_query
 
-    if token is None:
+    if not token:
         raise HTTPException(
-            status_code=status.HTTP_401_UNAUTHORIZED,
+            status_code=HTTPStatus.UNAUTHORIZED,
             detail="Invoice (or Admin) key required.",
         )
 

lnbits/extensions/lnaddress/templates/lnaddress/_api_docs.html

@@ -13,7 +13,7 @@
         Charge people for using your domain name...<br />
 
         <a
-          href="https://github.com/lnbits/lnbits/tree/master/lnbits/extensions/lnaddress"
+          href="https://github.com/lnbits/lnbits-legend/tree/main/lnbits/extensions/lnaddress"
           >More details</a
         >
         <br />

lnbits/extensions/lnurlp/templates/lnurlp/index.html

@@ -296,16 +296,17 @@
         <q-btn
           outline
           color="grey"
+          icon="link"
           @click="copyText(qrCodeDialog.data.pay_url, 'Link copied to clipboard!')"
-          >Shareable link</q-btn
-        >
+          ><q-tooltip>Copy sharable link</q-tooltip>
+        </q-btn>
         <q-btn
           outline
           color="grey"
           icon="nfc"
           @click="writeNfcTag(qrCodeDialog.data.lnurl)"
           :disable="nfcTagWriting"
-        >
+          ><q-tooltip>Write to NFC</q-tooltip>
         </q-btn>
         <q-btn
           outline
@@ -314,7 +315,8 @@
           type="a"
           :href="qrCodeDialog.data.print_url"
           target="_blank"
-        ></q-btn>
+          ><q-tooltip>Print</q-tooltip></q-btn
+        >
         <q-btn v-close-popup flat color="grey" class="q-ml-auto">Close</q-btn>
       </div>
     </q-card>

lnbits/extensions/satsdice/templates/satsdice/index.html

@@ -232,7 +232,7 @@
         <q-btn
           outline
           color="grey"
-          icon="share"
+          icon="link"
           @click="copyText(qrCodeDialog.data.pay_url, 'Link copied to clipboard!')"
           ><q-tooltip>Copy shareable link</q-tooltip></q-btn
         >

lnbits/extensions/subdomains/templates/subdomains/_api_docs.html

@@ -13,7 +13,7 @@
         Charge people for using your subdomain name...<br />
 
         <a
-          href="https://github.com/lnbits/lnbits/tree/master/lnbits/extensions/subdomains"
+          href="https://github.com/lnbits/lnbits-legend/tree/main/lnbits/extensions/subdomains"
           >More details</a
         >
         <br />

lnbits/extensions/tipjar/crud.py

@@ -76,10 +76,10 @@ async def get_tipjars(wallet_id: str) -> Optional[list]:
 
 async def delete_tipjar(tipjar_id: int) -> None:
     """Delete a TipJar and all corresponding Tips"""
-    await db.execute("DELETE FROM tipjar.TipJars WHERE id = ?", (tipjar_id,))
     rows = await db.fetchall("SELECT * FROM tipjar.Tips WHERE tipjar = ?", (tipjar_id,))
     for row in rows:
         await delete_tip(row["id"])
+    await db.execute("DELETE FROM tipjar.TipJars WHERE id = ?", (tipjar_id,))
 
 
 async def get_tip(tip_id: str) -> Optional[Tip]:

lnbits/extensions/withdraw/templates/withdraw/index.html

@@ -418,16 +418,18 @@
         <q-btn
           outline
           color="grey"
+          icon="link"
           @click="copyText(qrCodeDialog.data.withdraw_url, 'Link copied to clipboard!')"
-          >Shareable link</q-btn
-        >
+          ><q-tooltip>Copy sharable link</q-tooltip>
+        </q-btn>
         <q-btn
           outline
           color="grey"
           icon="nfc"
           @click="writeNfcTag(qrCodeDialog.data.lnurl)"
           :disable="nfcTagWriting"
-        ></q-btn>
+          ><q-tooltip>Write to NFC</q-tooltip></q-btn
+        >
         <q-btn
           outline
           color="grey"
@@ -435,7 +437,8 @@
           type="a"
           :href="qrCodeDialog.data.print_url"
           target="_blank"
-        ></q-btn>
+          ><q-tooltip>Print</q-tooltip></q-btn
+        >
         <q-btn v-close-popup flat color="grey" class="q-ml-auto">Close</q-btn>
       </div>
     </q-card>

lnbits/static/js/components.js

@@ -179,6 +179,11 @@ Vue.component('lnbits-extension-list', {
 
 Vue.component('lnbits-payment-details', {
   props: ['payment'],
+  data: function () {
+    return {
+      LNBITS_DENOMINATION: LNBITS_DENOMINATION
+    }
+  },
   template: `
     <div class="q-py-md" style="text-align: left">
       <div class="row justify-center q-mb-md">

tests/core/views/test_api.py

@@ -45,9 +45,16 @@ async def test_get_wallet_adminkey(client, adminkey_headers_to):
     assert "id" in result
 
 
-# check POST /api/v1/payments: empty request
+# check PUT /api/v1/wallet/newwallet: empty request where admin key is needed
 @pytest.mark.asyncio
-async def test_post_empty_request(client):
+async def test_put_empty_request_expected_admin_keys(client):
+    response = await client.put("/api/v1/wallet/newwallet")
+    assert response.status_code == 401
+
+
+# check POST /api/v1/payments: empty request where invoice key is needed
+@pytest.mark.asyncio
+async def test_post_empty_request_expected_invoice_keys(client):
     response = await client.post("/api/v1/payments")
     assert response.status_code == 401
 

tools/conv.py

@@ -19,16 +19,12 @@ env.read_env()
 # Change these values as needed
 
 
-sqfolder = "data/"
+sqfolder = env.str("LNBITS_DATA_FOLDER", default=None)
 
 LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None)
 if LNBITS_DATABASE_URL is None:
-    pgdb = "lnbits"
-    pguser = "lnbits"
-    pgpswd = "postgres"
-    pghost = "localhost"
-    pgport = "5432"
-    pgschema = ""
+    print("missing LNBITS_DATABASE_URL")
+    sys.exit(1)
 else:
     # parse postgres://lnbits:postgres@localhost:5432/lnbits
     pgdb = LNBITS_DATABASE_URL.split("/")[-1]
@@ -129,7 +125,7 @@ def migrate_db(file: str, schema: str, exclude_tables: List[str] = []):
     sq = get_sqlite_cursor(file)
     tables = sq.execute(
         """
-        SELECT name FROM sqlite_master 
+        SELECT name FROM sqlite_master
         WHERE type='table' AND name not like 'sqlite?_%' escape '?'
     """
     ).fetchall()