diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 8e72cf62..90006d2a 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -9,9 +9,9 @@ jobs: postgres: image: postgres:latest env: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres + POSTGRES_USER: lnbits + POSTGRES_PASSWORD: lnbits + POSTGRES_DB: migration ports: # maps tcp port 5432 on service container to the host - 5432:5432 @@ -36,11 +36,4 @@ jobs: sudo apt install unzip - name: Run migrations run: | - rm -rf ./data - mkdir -p ./data - export LNBITS_DATA_FOLDER="./data" - unzip tests/data/mock_data.zip -d ./data - timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi - export LNBITS_DATABASE_URL="postgres://postgres:postgres@0.0.0.0:5432/postgres" - timeout 5s poetry run lnbits --host 0.0.0.0 --port 5001 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi - poetry run python tools/conv.py + make test-migration diff --git a/Makefile b/Makefile index d91d0421..80ae00c2 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: format check requirements.txt format: prettier isort black -check: mypy checkprettier checkisort checkblack +check: mypy checkprettier checkisort checkblack prettier: $(shell find lnbits -name "*.js" -name ".html") ./node_modules/.bin/prettier --write lnbits/static/js/*.js lnbits/core/static/js/*.js lnbits/extensions/*/templates/*/*.html ./lnbits/core/templates/core/*.html lnbits/templates/*.html lnbits/extensions/*/static/js/*.js lnbits/extensions/*/static/components/*/*.js lnbits/extensions/*/static/components/*/*.html @@ -32,11 +32,13 @@ test: FAKE_WALLET_SECRET="ToTheMoon1" \ LNBITS_DATA_FOLDER="./tests/data" \ PYTHONUNBUFFERED=1 \ + DEBUG=true \ poetry run pytest test-real-wallet: LNBITS_DATA_FOLDER="./tests/data" \ PYTHONUNBUFFERED=1 \ + DEBUG=true \ poetry run pytest test-venv: @@ -44,7 +46,27 @@ test-venv: FAKE_WALLET_SECRET="ToTheMoon1" \ LNBITS_DATA_FOLDER="./tests/data" \ PYTHONUNBUFFERED=1 \ + DEBUG=true \ ./venv/bin/pytest --durations=1 -s --cov=lnbits --cov-report=xml tests +test-migration: + rm -rf ./migration-data + mkdir -p ./migration-data + unzip tests/data/mock_data.zip -d ./migration-data + HOST=0.0.0.0 \ + PORT=5002 \ + LNBITS_DATA_FOLDER="./migration-data" \ + timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + HOST=0.0.0.0 \ + PORT=5002 \ + LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \ + timeout 5s poetry run lnbits --host 0.0.0.0 --port 5002 || code=$?; if [[ $code -ne 124 && $code -ne 0 ]]; then exit $code; fi + LNBITS_DATA_FOLDER="./migration-data" \ + LNBITS_DATABASE_URL="postgres://lnbits:lnbits@localhost:5432/migration" \ + poetry run python tools/conv.py + +migration: + poetry run python tools/conv.py + bak: # LNBITS_DATABASE_URL=postgres://postgres:postgres@0.0.0.0:5432/postgres diff --git a/docs/devs/extensions.md b/docs/devs/extensions.md index 0ceb9cb3..cd81a021 100644 --- a/docs/devs/extensions.md +++ b/docs/devs/extensions.md @@ -48,4 +48,25 @@ LNbits currently supports SQLite and PostgreSQL databases. There is a migration ### Adding mock data to `mock_data.zip` -`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR. \ No newline at end of file +`mock_data.zip` contains a few lines of sample SQLite data and is used in automated GitHub test to see whether your migration in `conv.py` works. Run your extension and save a few lines of data into a SQLite `your_extension.sqlite3` file. Unzip `tests/data/mock_data.zip`, add `your_extension.sqlite3`, updated `database.sqlite3` and zip it again. Add the updated `mock_data.zip` to your PR. + +### running migration locally +you will need a running postgres database + +#### create lnbits user for migration database +```console +sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'" +``` +#### create migration database +```console +sudo su - postgres -c "psql -c 'CREATE DATABASE migration;'" +``` +#### run the migration +```console +make test-migration +``` +sudo su - postgres -c "psql -c 'CREATE ROLE lnbits LOGIN PASSWORD 'lnbits';'" +#### clean migration database afterwards, fails if you try again +```console +sudo su - postgres -c "psql -c 'DROP DATABASE IF EXISTS migration;'" +``` diff --git a/docs/guide/installation.md b/docs/guide/installation.md index 7c473eff..6de893ad 100644 --- a/docs/guide/installation.md +++ b/docs/guide/installation.md @@ -170,8 +170,9 @@ LNBITS_DATABASE_URL="postgres://postgres:postgres@localhost/lnbits" # START LNbits # STOP LNbits -# on the LNBits folder, locate and edit 'tools/conv.py' with the relevant credentials -python3 tools/conv.py +poetry run python tools/conv.py +# or +make migration ``` Hopefully, everything works and get migrated... Launch LNbits again and check if everything is working properly. @@ -194,15 +195,14 @@ Description=LNbits [Service] # replace with the absolute path of your lnbits installation -WorkingDirectory=/home/bitcoin/lnbits -# same here -ExecStart=/home/bitcoin/lnbits/venv/bin/uvicorn lnbits.__main__:app --port 5000 +WorkingDirectory=/home/lnbits/lnbits-legend +# same here. run `which poetry` if you can't find the poetry binary +ExecStart=/home/lnbits/.local/bin/poetry run lnbits # replace with the user that you're running lnbits on -User=bitcoin +User=lnbits Restart=always TimeoutSec=120 RestartSec=30 -# this makes sure that you receive logs in real time Environment=PYTHONUNBUFFERED=1 [Install] diff --git a/lnbits/core/static/js/wallet.js b/lnbits/core/static/js/wallet.js index baa9f605..15cab4b6 100644 --- a/lnbits/core/static/js/wallet.js +++ b/lnbits/core/static/js/wallet.js @@ -668,7 +668,17 @@ new Vue({ }) }, exportCSV: function () { - LNbits.utils.exportCSV(this.paymentsTable.columns, this.payments) + // status is important for export but it is not in paymentsTable + // because it is manually added with payment detail link and icons + // and would cause duplication in the list + let columns = this.paymentsTable.columns + columns.unshift({ + name: 'pending', + align: 'left', + label: 'Pending', + field: 'pending' + }) + LNbits.utils.exportCSV(columns, this.payments) } }, watch: { diff --git a/lnbits/core/views/api.py b/lnbits/core/views/api.py index 5a6d1140..d764d3d2 100644 --- a/lnbits/core/views/api.py +++ b/lnbits/core/views/api.py @@ -1,7 +1,7 @@ import asyncio +import binascii import hashlib import json -from binascii import unhexlify from http import HTTPStatus from io import BytesIO from typing import Dict, List, Optional, Tuple, Union @@ -152,11 +152,23 @@ class CreateInvoiceData(BaseModel): async def api_payments_create_invoice(data: CreateInvoiceData, wallet: Wallet): if data.description_hash: - description_hash = unhexlify(data.description_hash) + try: + description_hash = binascii.unhexlify(data.description_hash) + except binascii.Error: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="'description_hash' must be a valid hex string", + ) unhashed_description = b"" memo = "" elif data.unhashed_description: - unhashed_description = unhexlify(data.unhashed_description) + try: + unhashed_description = binascii.unhexlify(data.unhashed_description) + except binascii.Error: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="'unhashed_description' must be a valid hex string", + ) description_hash = b"" memo = "" else: diff --git a/lnbits/decorators.py b/lnbits/decorators.py index 090c11c5..69b26fe7 100644 --- a/lnbits/decorators.py +++ b/lnbits/decorators.py @@ -130,10 +130,13 @@ async def get_key_type( # 2: invalid pathname = r["path"].split("/")[1] - if not api_key_header and not api_key_query: - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST) + token = api_key_header or api_key_query - token = api_key_header if api_key_header else api_key_query + if not token: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail="Invoice (or Admin) key required.", + ) try: admin_checker = WalletAdminKeyChecker(api_key=token) @@ -180,7 +183,14 @@ async def require_admin_key( api_key_header: str = Security(api_key_header), # type: ignore api_key_query: str = Security(api_key_query), # type: ignore ): - token = api_key_header if api_key_header else api_key_query + + token = api_key_header or api_key_query + + if not token: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail="Admin key required.", + ) wallet = await get_key_type(r, token) @@ -199,11 +209,12 @@ async def require_invoice_key( api_key_header: str = Security(api_key_header), # type: ignore api_key_query: str = Security(api_key_query), # type: ignore ): + token = api_key_header or api_key_query - if token is None: + if not token: raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, + status_code=HTTPStatus.UNAUTHORIZED, detail="Invoice (or Admin) key required.", ) diff --git a/lnbits/extensions/lnaddress/templates/lnaddress/_api_docs.html b/lnbits/extensions/lnaddress/templates/lnaddress/_api_docs.html index 49155f27..12173c95 100644 --- a/lnbits/extensions/lnaddress/templates/lnaddress/_api_docs.html +++ b/lnbits/extensions/lnaddress/templates/lnaddress/_api_docs.html @@ -13,7 +13,7 @@ Charge people for using your domain name...
More details
diff --git a/lnbits/extensions/lnurlp/templates/lnurlp/index.html b/lnbits/extensions/lnurlp/templates/lnurlp/index.html index 9677a027..de90f5af 100644 --- a/lnbits/extensions/lnurlp/templates/lnurlp/index.html +++ b/lnbits/extensions/lnurlp/templates/lnurlp/index.html @@ -296,16 +296,17 @@ Shareable link + >Copy sharable link + + >Write to NFC + >Print Close diff --git a/lnbits/extensions/satsdice/templates/satsdice/index.html b/lnbits/extensions/satsdice/templates/satsdice/index.html index 004e9ccb..654069d8 100644 --- a/lnbits/extensions/satsdice/templates/satsdice/index.html +++ b/lnbits/extensions/satsdice/templates/satsdice/index.html @@ -232,7 +232,7 @@ Copy shareable link diff --git a/lnbits/extensions/subdomains/templates/subdomains/_api_docs.html b/lnbits/extensions/subdomains/templates/subdomains/_api_docs.html index db3b2477..8b3a4c4d 100644 --- a/lnbits/extensions/subdomains/templates/subdomains/_api_docs.html +++ b/lnbits/extensions/subdomains/templates/subdomains/_api_docs.html @@ -13,7 +13,7 @@ Charge people for using your subdomain name...
More details
diff --git a/lnbits/extensions/tipjar/crud.py b/lnbits/extensions/tipjar/crud.py index 29e1a469..046b9491 100644 --- a/lnbits/extensions/tipjar/crud.py +++ b/lnbits/extensions/tipjar/crud.py @@ -76,10 +76,10 @@ async def get_tipjars(wallet_id: str) -> Optional[list]: async def delete_tipjar(tipjar_id: int) -> None: """Delete a TipJar and all corresponding Tips""" - await db.execute("DELETE FROM tipjar.TipJars WHERE id = ?", (tipjar_id,)) rows = await db.fetchall("SELECT * FROM tipjar.Tips WHERE tipjar = ?", (tipjar_id,)) for row in rows: await delete_tip(row["id"]) + await db.execute("DELETE FROM tipjar.TipJars WHERE id = ?", (tipjar_id,)) async def get_tip(tip_id: str) -> Optional[Tip]: diff --git a/lnbits/extensions/withdraw/templates/withdraw/index.html b/lnbits/extensions/withdraw/templates/withdraw/index.html index b1d927af..27684f6b 100644 --- a/lnbits/extensions/withdraw/templates/withdraw/index.html +++ b/lnbits/extensions/withdraw/templates/withdraw/index.html @@ -418,16 +418,18 @@ Shareable link + >Copy sharable link + + >Write to NFC + >Print Close diff --git a/lnbits/static/js/components.js b/lnbits/static/js/components.js index b8c9f4d0..ab3f7f08 100644 --- a/lnbits/static/js/components.js +++ b/lnbits/static/js/components.js @@ -179,6 +179,11 @@ Vue.component('lnbits-extension-list', { Vue.component('lnbits-payment-details', { props: ['payment'], + data: function () { + return { + LNBITS_DENOMINATION: LNBITS_DENOMINATION + } + }, template: `
diff --git a/tests/core/views/test_api.py b/tests/core/views/test_api.py index 219762d3..86c17fa9 100644 --- a/tests/core/views/test_api.py +++ b/tests/core/views/test_api.py @@ -45,9 +45,16 @@ async def test_get_wallet_adminkey(client, adminkey_headers_to): assert "id" in result -# check POST /api/v1/payments: empty request +# check PUT /api/v1/wallet/newwallet: empty request where admin key is needed @pytest.mark.asyncio -async def test_post_empty_request(client): +async def test_put_empty_request_expected_admin_keys(client): + response = await client.put("/api/v1/wallet/newwallet") + assert response.status_code == 401 + + +# check POST /api/v1/payments: empty request where invoice key is needed +@pytest.mark.asyncio +async def test_post_empty_request_expected_invoice_keys(client): response = await client.post("/api/v1/payments") assert response.status_code == 401 diff --git a/tests/data/mock_data.zip b/tests/data/mock_data.zip index 3a793891..8704eb6d 100644 Binary files a/tests/data/mock_data.zip and b/tests/data/mock_data.zip differ diff --git a/tools/conv.py b/tools/conv.py index 10e8c9cd..5084660f 100644 --- a/tools/conv.py +++ b/tools/conv.py @@ -19,16 +19,12 @@ env.read_env() # Change these values as needed -sqfolder = "data/" +sqfolder = env.str("LNBITS_DATA_FOLDER", default=None) LNBITS_DATABASE_URL = env.str("LNBITS_DATABASE_URL", default=None) if LNBITS_DATABASE_URL is None: - pgdb = "lnbits" - pguser = "lnbits" - pgpswd = "postgres" - pghost = "localhost" - pgport = "5432" - pgschema = "" + print("missing LNBITS_DATABASE_URL") + sys.exit(1) else: # parse postgres://lnbits:postgres@localhost:5432/lnbits pgdb = LNBITS_DATABASE_URL.split("/")[-1] @@ -129,7 +125,7 @@ def migrate_db(file: str, schema: str, exclude_tables: List[str] = []): sq = get_sqlite_cursor(file) tables = sq.execute( """ - SELECT name FROM sqlite_master + SELECT name FROM sqlite_master WHERE type='table' AND name not like 'sqlite?_%' escape '?' """ ).fetchall()