From 38e5eeece02f71c5a78c812e0f69e244bc19c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:39:27 +0200 Subject: [PATCH 01/81] add the main tasks to scheduled_tasks and make dem deinitilize on api_stop --- __init__.py | 9 +++++++-- views_api.py | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index 60d8e23..5a251b0 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +from typing import List from fastapi import APIRouter from starlette.staticfiles import StaticFiles @@ -17,6 +18,8 @@ nostrclient_static_files = [ nostrclient_ext: APIRouter = APIRouter(prefix="/nostrclient", tags=["nostrclient"]) +scheduled_tasks: List[asyncio.Task] = [] + def nostr_renderer(): return template_renderer(["lnbits/extensions/nostrclient/templates"]) @@ -29,5 +32,7 @@ from .views_api import * # noqa def nostrclient_start(): loop = asyncio.get_event_loop() - loop.create_task(catch_everything_and_restart(init_relays)) - loop.create_task(catch_everything_and_restart(subscribe_events)) + task1 = loop.create_task(catch_everything_and_restart(init_relays)) + scheduled_tasks.append(task1) + task2 = loop.create_task(catch_everything_and_restart(subscribe_events)) + scheduled_tasks.append(task2) diff --git a/views_api.py b/views_api.py index 15cc3ab..c0be01e 100644 --- a/views_api.py +++ b/views_api.py @@ -9,7 +9,7 @@ from starlette.exceptions import HTTPException from lnbits.decorators import check_admin from lnbits.helpers import urlsafe_short_hash -from . import nostrclient_ext +from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays from .models import Relay, RelayList from .services import NostrRouter, nostr @@ -92,6 +92,12 @@ async def api_stop(): except Exception as e: logger.error(e) + for scheduled_task in scheduled_tasks: + try: + scheduled_task.cancel() + except Exception as ex: + logger.warning(ex) + return {"success": True} From 6500e5c9bb2df62468ee7c099294eb1edff8fbb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:42:14 +0200 Subject: [PATCH 02/81] forgot asyncio --- __init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/__init__.py b/__init__.py index 5a251b0..b09d0e9 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +import asyncio from typing import List from fastapi import APIRouter from starlette.staticfiles import StaticFiles From 625748e1199a593b23ee3828364dc733ee3ff2ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Mon, 17 Apr 2023 14:46:45 +0200 Subject: [PATCH 03/81] some import cleanup and wrong fastapi imports --- views.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/views.py b/views.py index a05f956..4214612 100644 --- a/views.py +++ b/views.py @@ -1,17 +1,9 @@ -import asyncio -from http import HTTPStatus - -# FastAPI good for incoming -from fastapi import Request -from fastapi.param_functions import Query -from fastapi.params import Depends +from fastapi import Request, Depends from fastapi.templating import Jinja2Templates from starlette.responses import HTMLResponse -from lnbits.core.crud import update_payment_status from lnbits.core.models import User -from lnbits.core.views.api import api_payment -from lnbits.decorators import check_admin, check_user_exists +from lnbits.decorators import check_admin from . import nostr_renderer, nostrclient_ext From 2814d277e5134d96b4732546c13a8eff0e90430a Mon Sep 17 00:00:00 2001 From: Tiago Vasconcelos Date: Mon, 24 Apr 2023 10:38:11 +0100 Subject: [PATCH 04/81] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5f9bfbc..7fbb640 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -# nostrclient +# Nostrclient - [LNbits](https://github.com/lnbits/lnbits) extension +For more about LNBits extension check [this tutorial](https://github.com/lnbits/lnbits/wiki/LNbits-Extensions) `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. From b5d98910db5ecfb53d8414e0197511e63d7477a8 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 10:12:49 +0300 Subject: [PATCH 05/81] chore: code format --- __init__.py | 1 + services.py | 1 + tasks.py | 4 ++-- views.py | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/__init__.py b/__init__.py index b09d0e9..019df68 100644 --- a/__init__.py +++ b/__init__.py @@ -1,5 +1,6 @@ import asyncio from typing import List + from fastapi import APIRouter from starlette.staticfiles import StaticFiles diff --git a/services.py b/services.py index 474bf90..82f6578 100644 --- a/services.py +++ b/services.py @@ -4,6 +4,7 @@ from typing import List, Union from fastapi import WebSocket, WebSocketDisconnect from loguru import logger + from lnbits.helpers import urlsafe_short_hash from .models import Event, Filter, Filters, Relay, RelayList diff --git a/tasks.py b/tasks.py index ab9a656..beff9db 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,6 @@ import asyncio -import ssl import json +import ssl import threading from .crud import get_relays @@ -11,8 +11,8 @@ from .nostr.relay_manager import RelayManager from .services import ( nostr, received_subscription_eosenotices, - received_subscription_notices, received_subscription_events, + received_subscription_notices, ) diff --git a/views.py b/views.py index 4214612..57b73a1 100644 --- a/views.py +++ b/views.py @@ -1,4 +1,4 @@ -from fastapi import Request, Depends +from fastapi import Depends, Request from fastapi.templating import Jinja2Templates from starlette.responses import HTMLResponse From dc6c218618f19aa2c0e85489c46555b0918f6514 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 10:25:09 +0300 Subject: [PATCH 06/81] feat: prepare UI --- templates/nostrclient/index.html | 58 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index abe5a87..41fef48 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -2,6 +2,24 @@ %} {% block page %} {% raw %}
+ + +
+
+ +
+
+ Add relay +
+
+
+
@@ -42,7 +60,7 @@
{{ col.label }}
- + @@ -76,36 +94,24 @@ - -
- Your endpoint: - -
-
+ - - -
-
- +
+
+
+ Your endpoint: + +
-
- Add relay +
+ Test Endpoint
-
From 977ee84d9e661aedd626d9434da2fbd7ce0fff43 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 11:09:54 +0300 Subject: [PATCH 07/81] feat: basic test UI --- templates/nostrclient/index.html | 86 +++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 41fef48..4c63400 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -97,7 +97,8 @@ -
+ +
Your endpoint: @@ -112,6 +113,83 @@ Test Endpoint
+ + + +
+
+ Private Key (optional): +
+
+ +
+
+
+
+
+
+ + This should be a temp private (throw away). No not user your own private key! + + + + It is optional. One can be generated for you! + +
+ +
+
+
+ Test Message: +
+
+ +
+ +
+
+
+ Public Key (hex or npub): +
+
+ +
+
+
+
+
+
+ + This is the recipient of the message + +
+
+
+
+ Send Message +
+
+
@@ -126,7 +204,6 @@

- Date: Fri, 5 May 2023 11:10:23 +0300 Subject: [PATCH 08/81] feat: ass basic test api --- models.py | 10 ++++++++++ views_api.py | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/models.py b/models.py index 4ed1e30..899359f 100644 --- a/models.py +++ b/models.py @@ -50,6 +50,16 @@ class Filters(BaseModel): __root__: List[Filter] +class TestMessage(BaseModel): + sender_private_key: Optional[str] + reciever_public_key: str + message: str + +class TestMessageResponse(BaseModel): + private_key: str + public_key: str + event: Event + # class nostrKeys(BaseModel): # pubkey: str # privkey: str diff --git a/views_api.py b/views_api.py index c0be01e..3287351 100644 --- a/views_api.py +++ b/views_api.py @@ -11,7 +11,7 @@ from lnbits.helpers import urlsafe_short_hash from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays -from .models import Relay, RelayList +from .models import Relay, RelayList, TestMessage, TestMessageResponse from .services import NostrRouter, nostr from .tasks import init_relays @@ -75,6 +75,26 @@ async def api_delete_relay(relay: Relay) -> None: await delete_relay(relay) +@nostrclient_ext.put( + "/api/v1/relay/test", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] +) +async def api_test_endpoint(test_message: TestMessage) -> TestMessageResponse: + try: + print("### api_test_endpoint", test_message) + except (ValueError, AssertionError) as ex: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail=str(ex), + ) + except Exception as ex: + logger.warning(ex) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail="Cannot generate test event", + ) + + + @nostrclient_ext.delete( "/api/v1", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) From 77f7c0b18f8a5b24c2bbff75ae305e9d77565013 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 11:12:12 +0300 Subject: [PATCH 09/81] chore: code format --- templates/nostrclient/index.html | 39 +++++++++++++++++++------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 4c63400..593419a 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -15,7 +15,9 @@ >

- Add relay + Add relay +
@@ -60,7 +62,6 @@
{{ col.label }}
- @@ -94,7 +95,6 @@ - @@ -110,7 +110,9 @@
- Test Endpoint + Test Endpoint
@@ -131,22 +133,23 @@
-
-
+
- This should be a temp private (throw away). No not user your own private key! + This should be a temp private (throw away). No not user your + own private key! + - - + + It is optional. One can be generated for you!
-
- Test Message: + Test Message:
-
@@ -176,8 +178,7 @@
-
-
+
This is the recipient of the message @@ -186,7 +187,13 @@
- Send Message + Send Message
@@ -253,7 +260,7 @@ testData: { senderPrivateKey: null, recieverPublicKey: null, - message: null, + message: null }, relayTable: { columns: [ From 5d906c1fda17e13a52881f1dcb9c9c11eabf35fa Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:00:09 +0300 Subject: [PATCH 10/81] feat: send simple DM --- helpers.py | 19 +++++++ models.py | 2 +- templates/nostrclient/index.html | 88 ++++++++++++++++++++++++++++++-- views_api.py | 19 ++++++- 4 files changed, 122 insertions(+), 6 deletions(-) create mode 100644 helpers.py diff --git a/helpers.py b/helpers.py new file mode 100644 index 0000000..bcf5c02 --- /dev/null +++ b/helpers.py @@ -0,0 +1,19 @@ +from bech32 import bech32_decode, convertbits + + +def normalize_public_key(pubkey: str) -> str: + if pubkey.startswith("npub1"): + _, decoded_data = bech32_decode(pubkey) + if not decoded_data: + raise ValueError("Public Key is not valid npub") + + decoded_data_bits = convertbits(decoded_data, 5, 8, False) + if not decoded_data_bits: + raise ValueError("Public Key is not valid npub") + return bytes(decoded_data_bits).hex() + + # check if valid hex + if len(pubkey) != 64: + raise ValueError("Public Key is not valid hex") + int(pubkey, 16) + return pubkey diff --git a/models.py b/models.py index 899359f..1456d83 100644 --- a/models.py +++ b/models.py @@ -58,7 +58,7 @@ class TestMessage(BaseModel): class TestMessageResponse(BaseModel): private_key: str public_key: str - event: Event + event_json: str # class nostrKeys(BaseModel): # pubkey: str diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 593419a..90859ae 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -159,7 +159,7 @@ filled rows="3" type="textarea" - label="Test Message" + label="Test Message *" >
@@ -181,7 +181,7 @@
- This is the recipient of the message + This is the recipient of the message. Field required.
@@ -189,6 +189,7 @@
+ + +
+
+ Sent Data: +
+
+ +
+
+
@@ -258,9 +278,12 @@ nostrrelayLinks: [], filter: '', testData: { + wsConnection: null, senderPrivateKey: null, recieverPublicKey: null, - message: null + message: null, + sentData: '', + receivedData: '' }, relayTable: { columns: [ @@ -368,6 +391,64 @@ LNbits.utils.notifyApiError(error) }) }, + sendTestMessage: async function(){ + try { + const {data} = await LNbits.api.request( + 'PUT', + '/nostrclient/api/v1/relay/test?usr=' + this.g.user.id, + this.g.user.wallets[0].adminkey, + { + sender_private_key: this.testData.senderPrivateKey, + reciever_public_key: this.testData.recieverPublicKey, + message: this.testData.message + } + ) + console.log('### data', data) + this.testData.senderPrivateKey = data.private_key + this.$q.localStorage.set('lnbits.nostrclient.senderPrivateKey', data.private_key || '') + const event = JSON.parse(data.event_json)[1] + console.log('### event', event) + this.sendDataToWebSocket(data.event_json) + } catch (error) { + LNbits.utils.notifyApiError(error) + } + }, + + sendDataToWebSocket: async function (data){ + try { + if (!this.testData.wsConnection) { + this.connectToWebsocket() + } + this.testData.wsConnection.send(data) + this.testData.sentData = data + '\n' + this.testData.sentData + } catch (error) { + this.$q.notify({ + timeout: 5000, + type: 'warning', + message: 'Failed to connect to websocket', + caption: `${error}` + }) + } + }, + connectToWebsocket: function () { + const scheme = location.protocol === 'http:' ? 'ws' : 'wss' + const port = location.port ? `:${location.port}` : '' + const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` + this.testData.wsConnection = new WebSocket(wsUrl) + wsConnection.onmessage = async e => { + // const data = JSON.parse(e.data) + console.log('### onmessage', e.data) + } + wsConnection.onerror = async e => { + // const data = JSON.parse(e.data) + console.log('### onerror', e.data) + } + wsConnection.onclose = async e => { + // const data = JSON.parse(e.data) + console.log('### onclose', e.data) + } + + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -377,6 +458,7 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) + this.testData.senderPrivateKey = this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '' } }) diff --git a/views_api.py b/views_api.py index 3287351..e7c5f06 100644 --- a/views_api.py +++ b/views_api.py @@ -1,4 +1,5 @@ import asyncio +import json from http import HTTPStatus from typing import Optional @@ -11,7 +12,9 @@ from lnbits.helpers import urlsafe_short_hash from . import nostrclient_ext, scheduled_tasks from .crud import add_relay, delete_relay, get_relays +from .helpers import normalize_public_key from .models import Relay, RelayList, TestMessage, TestMessageResponse +from .nostr.key import EncryptedDirectMessage, PrivateKey from .services import NostrRouter, nostr from .tasks import init_relays @@ -78,9 +81,21 @@ async def api_delete_relay(relay: Relay) -> None: @nostrclient_ext.put( "/api/v1/relay/test", status_code=HTTPStatus.OK, dependencies=[Depends(check_admin)] ) -async def api_test_endpoint(test_message: TestMessage) -> TestMessageResponse: +async def api_test_endpoint(data: TestMessage) -> TestMessageResponse: try: - print("### api_test_endpoint", test_message) + to_public_key = normalize_public_key(data.reciever_public_key) + + pk = bytes.fromhex(data.sender_private_key) if data.sender_private_key else None + private_key = PrivateKey(pk) + + dm = EncryptedDirectMessage( + recipient_pubkey=to_public_key, cleartext_content=data.message + ) + private_key.sign_event(dm) + + print("### api_test_endpoint", data) + + return TestMessageResponse(private_key=private_key.hex(), public_key=to_public_key, event_json=dm.to_message()) except (ValueError, AssertionError) as ex: raise HTTPException( status_code=HTTPStatus.BAD_REQUEST, From d74776ee9324898c2d1bfc27714aa1ef42dbc8a4 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:01:07 +0300 Subject: [PATCH 11/81] chore: code format --- templates/nostrclient/index.html | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 90859ae..acf50af 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -391,7 +391,7 @@ LNbits.utils.notifyApiError(error) }) }, - sendTestMessage: async function(){ + sendTestMessage: async function () { try { const {data} = await LNbits.api.request( 'PUT', @@ -405,7 +405,10 @@ ) console.log('### data', data) this.testData.senderPrivateKey = data.private_key - this.$q.localStorage.set('lnbits.nostrclient.senderPrivateKey', data.private_key || '') + this.$q.localStorage.set( + 'lnbits.nostrclient.senderPrivateKey', + data.private_key || '' + ) const event = JSON.parse(data.event_json)[1] console.log('### event', event) this.sendDataToWebSocket(data.event_json) @@ -414,14 +417,14 @@ } }, - sendDataToWebSocket: async function (data){ + sendDataToWebSocket: async function (data) { try { - if (!this.testData.wsConnection) { - this.connectToWebsocket() - } - this.testData.wsConnection.send(data) - this.testData.sentData = data + '\n' + this.testData.sentData - } catch (error) { + if (!this.testData.wsConnection) { + this.connectToWebsocket() + } + this.testData.wsConnection.send(data) + this.testData.sentData = data + '\n' + this.testData.sentData + } catch (error) { this.$q.notify({ timeout: 5000, type: 'warning', @@ -447,8 +450,7 @@ // const data = JSON.parse(e.data) console.log('### onclose', e.data) } - - }, + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -458,7 +460,9 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) - this.testData.senderPrivateKey = this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '' + this.testData.senderPrivateKey = + this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || + '' } }) From 58772043d4c874eb0a6d35130cba1d18caf386c8 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:45:45 +0300 Subject: [PATCH 12/81] feat: show sent data --- templates/nostrclient/index.html | 43 +++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index acf50af..27a2e08 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -216,6 +216,22 @@ > +
+
+ Received Data: +
+
+ +
+
@@ -411,7 +427,7 @@ ) const event = JSON.parse(data.event_json)[1] console.log('### event', event) - this.sendDataToWebSocket(data.event_json) + await this.sendDataToWebSocket(data.event_json) } catch (error) { LNbits.utils.notifyApiError(error) } @@ -421,9 +437,11 @@ try { if (!this.testData.wsConnection) { this.connectToWebsocket() + await this.sleep(500) } this.testData.wsConnection.send(data) - this.testData.sentData = data + '\n' + this.testData.sentData + const separator = '='.repeat(80) + this.testData.sentData = data + `\n\n${separator}\n` + this.testData.sentData } catch (error) { this.$q.notify({ timeout: 5000, @@ -438,23 +456,20 @@ const port = location.port ? `:${location.port}` : '' const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` this.testData.wsConnection = new WebSocket(wsUrl) - wsConnection.onmessage = async e => { - // const data = JSON.parse(e.data) - console.log('### onmessage', e.data) - } - wsConnection.onerror = async e => { - // const data = JSON.parse(e.data) - console.log('### onerror', e.data) - } - wsConnection.onclose = async e => { - // const data = JSON.parse(e.data) - console.log('### onclose', e.data) + const updateReciveData = async e => { + console.log('### updateReciveData', e.data) + this.testData.receivedData = e.data + '\n' +this.testData.receivedData } + + this.testData.wsConnection.onmessage = updateReciveData + this.testData.wsConnection.onerror = updateReciveData + this.testData.wsConnection.onclose = updateReciveData }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) - } + }, + sleep: (ms) => new Promise(r => setTimeout(r, ms)) }, created: function () { var self = this From 81e0aa26f8ef424c7fb527e22bbe3eb06b57b37d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 13:58:43 +0300 Subject: [PATCH 13/81] feat: subscribe to DMs --- templates/nostrclient/index.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 27a2e08..2d5a92d 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -210,7 +210,6 @@ v-model="testData.sentData" dense filled - readonly rows="5" type="textarea" > @@ -226,7 +225,6 @@ v-model="testData.receivedData" dense filled - readonly rows="5" type="textarea" > @@ -428,6 +426,8 @@ const event = JSON.parse(data.event_json)[1] console.log('### event', event) await this.sendDataToWebSocket(data.event_json) + const subscription = JSON.stringify(["REQ", "test-dms", { "kinds": [4], "#p": [event.pubkey]}]) + this.testData.wsConnection.send(subscription) } catch (error) { LNbits.utils.notifyApiError(error) } @@ -457,8 +457,8 @@ const wsUrl = `${scheme}://${document.domain}${port}/nostrclient/api/v1/relay` this.testData.wsConnection = new WebSocket(wsUrl) const updateReciveData = async e => { - console.log('### updateReciveData', e.data) - this.testData.receivedData = e.data + '\n' +this.testData.receivedData + const separator = '='.repeat(80) + this.testData.receivedData = e.data + `\n\n${separator}\n` +this.testData.receivedData } this.testData.wsConnection.onmessage = updateReciveData From d2e1dcc2ee382ea5b02337a2d95e3785202a27a6 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 14:00:03 +0300 Subject: [PATCH 14/81] chore: code cleanup & format --- templates/nostrclient/index.html | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 2d5a92d..78dd952 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -417,16 +417,18 @@ message: this.testData.message } ) - console.log('### data', data) this.testData.senderPrivateKey = data.private_key this.$q.localStorage.set( 'lnbits.nostrclient.senderPrivateKey', data.private_key || '' ) const event = JSON.parse(data.event_json)[1] - console.log('### event', event) await this.sendDataToWebSocket(data.event_json) - const subscription = JSON.stringify(["REQ", "test-dms", { "kinds": [4], "#p": [event.pubkey]}]) + const subscription = JSON.stringify([ + 'REQ', + 'test-dms', + {kinds: [4], '#p': [event.pubkey]} + ]) this.testData.wsConnection.send(subscription) } catch (error) { LNbits.utils.notifyApiError(error) @@ -441,7 +443,8 @@ } this.testData.wsConnection.send(data) const separator = '='.repeat(80) - this.testData.sentData = data + `\n\n${separator}\n` + this.testData.sentData + this.testData.sentData = + data + `\n\n${separator}\n` + this.testData.sentData } catch (error) { this.$q.notify({ timeout: 5000, @@ -458,9 +461,10 @@ this.testData.wsConnection = new WebSocket(wsUrl) const updateReciveData = async e => { const separator = '='.repeat(80) - this.testData.receivedData = e.data + `\n\n${separator}\n` +this.testData.receivedData + this.testData.receivedData = + e.data + `\n\n${separator}\n` + this.testData.receivedData } - + this.testData.wsConnection.onmessage = updateReciveData this.testData.wsConnection.onerror = updateReciveData this.testData.wsConnection.onclose = updateReciveData @@ -469,7 +473,7 @@ var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) }, - sleep: (ms) => new Promise(r => setTimeout(r, ms)) + sleep: ms => new Promise(r => setTimeout(r, ms)) }, created: function () { var self = this From b748dc3cb0ac604f81723bde18ab931dd404d569 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 5 May 2023 14:27:07 +0300 Subject: [PATCH 15/81] chore: code clean-up --- templates/nostrclient/index.html | 80 +++++++++++++++++++++++++++----- views_api.py | 2 - 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 78dd952..bbd8f23 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -110,17 +110,17 @@
- Test Endpoint
- - + +
- Private Key (optional): + Sender Private Key:
+
+
+ Sender Public Key: +
+
+ +
+
Test Message: @@ -165,7 +179,7 @@
- Public Key (hex or npub): + Receiver Public Key:
@@ -198,8 +212,8 @@
- - + +
Sent Data: @@ -292,8 +306,10 @@ nostrrelayLinks: [], filter: '', testData: { + show: false, wsConnection: null, senderPrivateKey: null, + senderPublicKey: null, recieverPublicKey: null, message: null, sentData: '', @@ -405,6 +421,38 @@ LNbits.utils.notifyApiError(error) }) }, + toggleTestPanel: async function() { + if (this.testData.show) { + await this.hideTestPannel() + } else { + await this.showTestPanel() + } + }, + showTestPanel: async function() { + this.testData = { + show: true, + wsConnection: null, + senderPrivateKey: this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '', + recieverPublicKey: null, + message: null, + sentData: '', + receivedData: '' + } + await this.closeWebsocket() + this.connectToWebsocket() + }, + hideTestPannel: async function() { + await this.closeWebsocket() + this.testData = { + show: false, + wsConnection: null, + senderPrivateKey: null, + recieverPublicKey: null, + message: null, + sentData: '', + receivedData: '' + } + }, sendTestMessage: async function () { try { const {data} = await LNbits.api.request( @@ -423,6 +471,7 @@ data.private_key || '' ) const event = JSON.parse(data.event_json)[1] + this.testData.senderPublicKey = event.pubkey await this.sendDataToWebSocket(data.event_json) const subscription = JSON.stringify([ 'REQ', @@ -469,6 +518,16 @@ this.testData.wsConnection.onerror = updateReciveData this.testData.wsConnection.onclose = updateReciveData }, + closeWebsocket: async function () { + try { + if (this.testData.wsConnection) { + this.testData.wsConnection.close() + await this.sleep(100) + } + } catch (error) { + console.warn(error) + } + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) @@ -479,9 +538,6 @@ var self = this this.getRelays() setInterval(this.getRelays, 5000) - this.testData.senderPrivateKey = - this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || - '' } }) diff --git a/views_api.py b/views_api.py index e7c5f06..12f4f79 100644 --- a/views_api.py +++ b/views_api.py @@ -93,8 +93,6 @@ async def api_test_endpoint(data: TestMessage) -> TestMessageResponse: ) private_key.sign_event(dm) - print("### api_test_endpoint", data) - return TestMessageResponse(private_key=private_key.hex(), public_key=to_public_key, event_json=dm.to_message()) except (ValueError, AssertionError) as ex: raise HTTPException( From ebe16b8993c852b1fd3b11737eb924debdcadc14 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 8 May 2023 11:57:56 +0300 Subject: [PATCH 16/81] doc: add Troubleshoot --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 7fbb640..1de61ba 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,12 @@ `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) + + +### Troubleshoot +The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. + +The LNbits user can DM itself (or a temp account) from `nostrclient` and verify that the messages are sent and received correctly. + +https://user-images.githubusercontent.com/2951406/236780745-929c33c2-2502-49be-84a3-db02a7aabc0e.mp4 + From 55b84de873cb1b1e18c9edefe10889a3efac17be Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 8 May 2023 12:16:41 +0200 Subject: [PATCH 17/81] expansion button and copy button --- templates/nostrclient/index.html | 281 ++++++++++++++++--------------- 1 file changed, 147 insertions(+), 134 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index bbd8f23..3db09f3 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -99,8 +99,18 @@
-
+
+ Copy address Your endpoint:
-
- -
- - -
-
- Sender Private Key: + + + +
+
+ Sender Private Key: +
+
+ +
-
- +
+
+
+ + + No not use your real private key! Leave empty for a randomly + generated key. + +
-
-
-
-
- - This should be a temp private (throw away). No not user your - own private key! - - +
+
+ Sender Public Key: +
+
+ +
+
+
+
+ Test Message: +
+
+ +
+
+
+
+ Receiver Public Key: +
+
+ +
+
+
+
+
+ + This is the recipient of the message. Field required. + +
+
+
+
+ Send Message +
+
+ - - It is optional. One can be generated for you! - + + +
+
+ Sent Data: +
+
+ +
-
-
-
- Sender Public Key: +
+
+ Received Data: +
+
+ +
-
- -
-
-
-
- Test Message: -
-
- -
-
-
-
- Receiver Public Key: -
-
- -
-
-
-
-
- - This is the recipient of the message. Field required. - -
-
-
-
- Send Message -
-
- - - -
-
- Sent Data: -
-
- -
-
-
-
- Received Data: -
-
- -
-
-
+ +
@@ -421,18 +431,21 @@ LNbits.utils.notifyApiError(error) }) }, - toggleTestPanel: async function() { + toggleTestPanel: async function () { if (this.testData.show) { await this.hideTestPannel() } else { await this.showTestPanel() } }, - showTestPanel: async function() { + showTestPanel: async function () { this.testData = { show: true, wsConnection: null, - senderPrivateKey: this.$q.localStorage.getItem('lnbits.nostrclient.senderPrivateKey') || '', + senderPrivateKey: + this.$q.localStorage.getItem( + 'lnbits.nostrclient.senderPrivateKey' + ) || '', recieverPublicKey: null, message: null, sentData: '', @@ -441,7 +454,7 @@ await this.closeWebsocket() this.connectToWebsocket() }, - hideTestPannel: async function() { + hideTestPannel: async function () { await this.closeWebsocket() this.testData = { show: false, From f27b4fa56962245b8abc37ef051c0bfe892c40b8 Mon Sep 17 00:00:00 2001 From: callebtc <93376500+callebtc@users.noreply.github.com> Date: Mon, 8 May 2023 12:29:43 +0200 Subject: [PATCH 18/81] readd toggletestbutton click --- templates/nostrclient/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 3db09f3..10fd4a5 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -125,6 +125,7 @@ group="advanced" icon="settings" label="Test this endpoint" + @click="toggleTestPanel" > From 629aa3a6c39de618ecb17fa5ecfe9b98103d8a85 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:54:04 +0300 Subject: [PATCH 19/81] fix: event uniqueness (see comment in code) --- nostr/message_pool.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/nostr/message_pool.py b/nostr/message_pool.py index d364cf2..578d673 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -1,8 +1,9 @@ import json from queue import Queue from threading import Lock -from .message_type import RelayMessageType + from .event import Event +from .message_type import RelayMessageType class EventMessage: @@ -69,9 +70,18 @@ class MessagePool: ) with self.lock: if not event.id in self._unique_events: - self.events.put(EventMessage(event, subscription_id, url)) - self._unique_events.add(event.id) + self._accept_event(EventMessage(event, subscription_id, url)) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) elif message_type == RelayMessageType.END_OF_STORED_EVENTS: self.eose_notices.put(EndOfStoredEventsMessage(message_json[1], url)) + + def _accept_event(self, event_message: EventMessage): + """ + Event uniqueness is considered per `subscription_id`. + The `subscription_id` is rewritten to be unique and it is the same accross relays. + The same event can come from different subscriptions (from the same client or from different ones). + Clients that have joined later should receive older events. + """ + self.events.put(event_message) + self._unique_events.add(f"{event_message.subscription_id}_{event_message.event.id}") \ No newline at end of file From 6852a9aa5ef25688eff7443b9c9e6f233a79ac27 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:55:41 +0300 Subject: [PATCH 20/81] fix: do not share the `subscriptions` object between relays Change in one relay reflects in others. The RelayManager takes care of updating each relay individually. --- nostr/relay_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index a698a33..fb5839f 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -23,7 +23,7 @@ class RelayManager: self, url: str, read: bool = True, write: bool = True, subscriptions={} ): policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions) + relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay def remove_relay(self, url: str): From 727f8fc3ce90451c1173fbca2165fa1bb1e2c695 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 11:55:56 +0300 Subject: [PATCH 21/81] fix: correctly handle `"REQ"` --- services.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/services.py b/services.py index 82f6578..b270539 100644 --- a/services.py +++ b/services.py @@ -48,16 +48,16 @@ class NostrRouter: break # registers a subscription if the input was a REQ request - subscription_id, json_str_rewritten = await self._add_nostr_subscription( + subscription_id, json_str_rewritten = await self._handle_nostr_subscription( json_str ) if subscription_id and json_str_rewritten: self.subscriptions.append(subscription_id) - json_str = json_str_rewritten # publish data - nostr.client.relay_manager.publish_message(json_str) + publish_data = json_str_rewritten or json_str + nostr.client.relay_manager.publish_message(publish_data) async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -139,7 +139,7 @@ class NostrRouter: ) return NostrFilters(filter_list) - async def _add_nostr_subscription(self, json_str): + async def _handle_nostr_subscription(self, json_str): """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect @@ -147,7 +147,7 @@ class NostrRouter: """ json_data = json.loads(json_str) assert len(json_data) - if json_data[0] in ["REQ", "CLOSE"]: + if json_data[0] == "REQ": subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.oridinal_subscription_ids[subscription_id_rewritten] = subscription_id @@ -160,4 +160,12 @@ class NostrRouter: [json_data[0], subscription_id_rewritten, fltr] ) return subscription_id_rewritten, request_rewritten + elif json_data[0] == "CLOSE": + subscription_id = json_data[1] + subscription_id_rewritten = next((k for k, v in self.oridinal_subscription_ids.items() if v == subscription_id), None) + if subscription_id_rewritten: + nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten]) + return None, request_rewritten + return None, None From 7e2c9033346cb969c674801cda45dc7cfd968eaa Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 31 May 2023 12:13:57 +0300 Subject: [PATCH 22/81] fix: uniqueness check --- nostr/message_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 578d673..02f7fd4 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -69,7 +69,7 @@ class MessagePool: e["sig"], ) with self.lock: - if not event.id in self._unique_events: + if not f"{subscription_id}_{event.id}" in self._unique_events: self._accept_event(EventMessage(event, subscription_id, url)) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) From 322679e7c5fe07b1004f6fb910e304dc81947651 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 21 Jun 2023 14:40:28 +0300 Subject: [PATCH 23/81] fix: do not use lambda in a loop --- nostr/pow.py | 2 ++ nostr/relay.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nostr/pow.py b/nostr/pow.py index e006288..fece484 100644 --- a/nostr/pow.py +++ b/nostr/pow.py @@ -1,7 +1,9 @@ import time + from .event import Event from .key import PrivateKey + def zero_bits(b: int) -> int: n = 0 diff --git a/nostr/relay.py b/nostr/relay.py index 7fb4baa..246b985 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,7 +2,9 @@ import json import time from queue import Queue from threading import Lock + from websocket import WebSocketApp + from .event import Event from .filter import Filters from .message_pool import MessagePool @@ -45,6 +47,7 @@ class Relay: self.queue = Queue() def connect(self, ssl_options: dict = None, proxy: dict = None): + print("### relay.connect", self.url) self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -81,24 +84,29 @@ class Relay: @property def ping(self): + print("### ping: ", self.url) ping_ms = int((self.ws.last_pong_tm - self.ws.last_ping_tm) * 1000) return ping_ms if self.connected and ping_ms > 0 else 0 def publish(self, message: str): self.queue.put(message) - def queue_worker(self, shutdown): + def queue_worker(self): + print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: if self.connected: try: message = self.queue.get(timeout=1) + print("#### queue_worker", message) self.num_sent_events += 1 self.ws.send(message) - except: - if shutdown(): + except Exception as e: + if self.shutdown: + print("#### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! e [", e, self.url, self.shutdown," ]###") break else: time.sleep(0.1) + print("#### OUT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) def add_subscription(self, id, filters: Filters): with self.lock: From d08e91b2c7b7954386141ee2b242b3f56da55fe0 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 21 Jun 2023 17:12:26 +0300 Subject: [PATCH 24/81] fix: do not double re-connect --- nostr/relay_manager.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index fb5839f..f6eba36 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -22,6 +22,8 @@ class RelayManager: def add_relay( self, url: str, read: bool = True, write: bool = True, subscriptions={} ): + if url in self.relays: + return policy = RelayPolicy(read, write) relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay @@ -42,21 +44,23 @@ class RelayManager: def open_connections(self, ssl_options: dict = None, proxy: dict = None): for relay in self.relays.values(): - self.threads[relay.url] = threading.Thread( - target=relay.connect, - args=(ssl_options, proxy), - name=f"{relay.url}-thread", - daemon=True, - ) - self.threads[relay.url].start() + if relay.url not in self.threads: + self.threads[relay.url] = threading.Thread( + target=relay.connect, + args=(ssl_options, proxy), + name=f"{relay.url}-thread", + daemon=True, + ) - self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, - args=(lambda: relay.shutdown,), - name=f"{relay.url}-queue", - daemon=True, - ) - self.queue_threads[relay.url].start() + self.threads[relay.url].start() + + if relay.url not in self.queue_threads: + self.queue_threads[relay.url] = threading.Thread( + target=relay.queue_worker, + name=f"{relay.url}-queue", + daemon=True, + ) + self.queue_threads[relay.url].start() def close_connections(self): for relay in self.relays.values(): From 09d2fc0493d56e407fa4b2c36d7fa0ba7fa6f672 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:02:15 +0300 Subject: [PATCH 25/81] refactor: `add_relay` logic --- nostr/client/client.py | 27 +++++++++------------ nostr/relay.py | 6 +++++ nostr/relay_manager.py | 55 ++++++++++++++++++++++++------------------ tasks.py | 20 +-------------- views_api.py | 14 ++++++++--- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 6e70f71..4d10647 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,19 +1,15 @@ -from typing import * -import ssl -import time +import base64 import json import os -import base64 - -from ..event import Event -from ..relay_manager import RelayManager -from ..message_type import ClientMessageType -from ..key import PrivateKey, PublicKey +import time +from typing import * +from ..event import EncryptedDirectMessage, Event, EventKind from ..filter import Filter, Filters -from ..event import Event, EventKind, EncryptedDirectMessage -from ..relay_manager import RelayManager +from ..key import PrivateKey, PublicKey from ..message_type import ClientMessageType +from ..relay_manager import RelayManager +from ..subscription import Subscription # from aes import AESCipher from . import cbc @@ -38,12 +34,11 @@ class NostrClient: if connect: self.connect() - def connect(self): + async def connect(self, subscriptions: dict[str, Subscription] = {}): for relay in self.relays: - self.relay_manager.add_relay(relay) - self.relay_manager.open_connections( - {"cert_reqs": ssl.CERT_NONE} - ) # NOTE: This disables ssl certificate verification + self.relay_manager.add_relay(relay, subscriptions) + + def close(self): self.relay_manager.close_connections() diff --git a/nostr/relay.py b/nostr/relay.py index 246b985..b6207b5 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -91,6 +91,12 @@ class Relay: def publish(self, message: str): self.queue.put(message) + def publish_subscriptions(self): + for _, subscription in self.subscriptions.items(): + s = subscription.to_json_object() + json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) + self.publish(json_str) + def queue_worker(self): print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f6eba36..0ec324a 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,11 +1,12 @@ -import json + +import ssl import threading from .event import Event from .filter import Filters from .message_pool import MessagePool -from .message_type import ClientMessageType from .relay import Relay, RelayPolicy +from .subscription import Subscription class RelayException(Exception): @@ -20,19 +21,30 @@ class RelayManager: self.message_pool = MessagePool() def add_relay( - self, url: str, read: bool = True, write: bool = True, subscriptions={} - ): + self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} + ) -> Relay: if url in self.relays: return + policy = RelayPolicy(read, write) relay = Relay(url, policy, self.message_pool, subscriptions.copy()) self.relays[url] = relay + self.open_connection( + relay, + {"cert_reqs": ssl.CERT_NONE} + ) # NOTE: This disables ssl certificate verification + + relay.publish_subscriptions() + return relay + def remove_relay(self, url: str): - self.relays[url].close() - self.relays.pop(url) self.threads[url].join(timeout=1) self.threads.pop(url) + self.queue_threads[url].join(timeout=1) + self.queue_threads.pop(url) + self.relays[url].close() + self.relays.pop(url) def add_subscription(self, id: str, filters: Filters): for relay in self.relays.values(): @@ -42,25 +54,22 @@ class RelayManager: for relay in self.relays.values(): relay.close_subscription(id) - def open_connections(self, ssl_options: dict = None, proxy: dict = None): - for relay in self.relays.values(): - if relay.url not in self.threads: - self.threads[relay.url] = threading.Thread( - target=relay.connect, - args=(ssl_options, proxy), - name=f"{relay.url}-thread", - daemon=True, - ) - self.threads[relay.url].start() + def open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + self.threads[relay.url] = threading.Thread( + target=relay.connect, + args=(ssl_options, proxy), + name=f"{relay.url}-thread", + daemon=True, + ) + self.threads[relay.url].start() - if relay.url not in self.queue_threads: - self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, - name=f"{relay.url}-queue", - daemon=True, - ) - self.queue_threads[relay.url].start() + self.queue_threads[relay.url] = threading.Thread( + target=relay.queue_worker, + name=f"{relay.url}-queue", + daemon=True, + ) + self.queue_threads[relay.url].start() def close_connections(self): for relay in self.relays.values(): diff --git a/tasks.py b/tasks.py index beff9db..eb5391a 100644 --- a/tasks.py +++ b/tasks.py @@ -17,31 +17,13 @@ from .services import ( async def init_relays(): - # we save any subscriptions teporarily to re-add them after reinitializing the client - subscriptions = {} - for relay in nostr.client.relay_manager.relays.values(): - # relay.add_subscription(id, filters) - for subscription_id, filters in relay.subscriptions.items(): - subscriptions[subscription_id] = filters - # reinitialize the entire client nostr.__init__() # get relays from db relays = await get_relays() # set relays and connect to them nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) - nostr.client.connect() - - await asyncio.sleep(2) - # re-add subscriptions - for subscription_id, subscription in subscriptions.items(): - nostr.client.relay_manager.add_subscription( - subscription_id, subscription.filters - ) - s = subscription.to_json_object() - json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) - nostr.client.relay_manager.publish_message(json_str) - return + await nostr.client.connect() async def subscribe_events(): diff --git a/views_api.py b/views_api.py index 12f4f79..88193ad 100644 --- a/views_api.py +++ b/views_api.py @@ -1,7 +1,7 @@ import asyncio import json from http import HTTPStatus -from typing import Optional +from typing import List, Optional from fastapi import Depends, WebSocket from loguru import logger @@ -15,6 +15,7 @@ from .crud import add_relay, delete_relay, get_relays from .helpers import normalize_public_key from .models import Relay, RelayList, TestMessage, TestMessageResponse from .nostr.key import EncryptedDirectMessage, PrivateKey +from .nostr.relay import Relay as NostrRelay from .services import NostrRouter, nostr from .tasks import init_relays @@ -60,8 +61,15 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: ) relay.id = urlsafe_short_hash() await add_relay(relay) - # we can't add relays during runtime yet - await init_relays() + + all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() + if len(all_relays): + subscriptions = all_relays[0].subscriptions + nostr.client.relays.append(relay.url) + nostr.client.relay_manager.add_relay(subscriptions) + + nostr.client.relay_manager.connect_relay(relay.url) + return await get_relays() From 1a58f0fea8a6340bbce2d910a05410efd2b326cb Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:02:29 +0300 Subject: [PATCH 26/81] chore: code format --- nostr/bech32.py | 1 + nostr/event.py | 7 ++++--- nostr/key.py | 13 +++++++------ nostr/subscription.py | 1 + 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/nostr/bech32.py b/nostr/bech32.py index b068de7..61a92c4 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -23,6 +23,7 @@ from enum import Enum + class Encoding(Enum): """Enumeration type to list the various supported encodings.""" BECH32 = 1 diff --git a/nostr/event.py b/nostr/event.py index b903e0e..65b187d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -1,10 +1,11 @@ -import time import json +import time from dataclasses import dataclass, field from enum import IntEnum -from typing import List -from secp256k1 import PublicKey from hashlib import sha256 +from typing import List + +from secp256k1 import PublicKey from .message_type import ClientMessageType diff --git a/nostr/key.py b/nostr/key.py index d34697f..8089e11 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,14 +1,15 @@ -import secrets import base64 -import secp256k1 -from cffi import FFI -from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes -from cryptography.hazmat.primitives import padding +import secrets from hashlib import sha256 +import secp256k1 +from cffi import FFI +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes + +from . import bech32 from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind -from . import bech32 class PublicKey: diff --git a/nostr/subscription.py b/nostr/subscription.py index 7afba20..76da0af 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,5 +1,6 @@ from .filter import Filters + class Subscription: def __init__(self, id: str, filters: Filters=None) -> None: self.id = id From 666009720a6b8189c8a85c49d2a828220f5801ec Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 10:22:13 +0300 Subject: [PATCH 27/81] chore: code clean-up --- nostr/client/client.py | 97 +----------------------------------------- nostr/relay.py | 5 --- tasks.py | 1 - 3 files changed, 1 insertion(+), 102 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 4d10647..faf5722 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,19 +1,9 @@ -import base64 -import json -import os import time from typing import * -from ..event import EncryptedDirectMessage, Event, EventKind -from ..filter import Filter, Filters -from ..key import PrivateKey, PublicKey -from ..message_type import ClientMessageType from ..relay_manager import RelayManager from ..subscription import Subscription -# from aes import AESCipher -from . import cbc - class NostrClient: relays = [ @@ -23,12 +13,8 @@ class NostrClient: "wss://nostr.oxtr.dev", ] # ["wss://nostr.oxtr.dev"] # ["wss://relay.nostr.info"] "wss://nostr-pub.wellorder.net" "ws://91.237.88.218:2700", "wss://nostrrr.bublina.eu.org", ""wss://nostr-relay.freeberty.net"", , "wss://nostr.oxtr.dev", "wss://relay.nostr.info", "wss://nostr-pub.wellorder.net" , "wss://relayer.fiatjaf.com", "wss://nodestr.fmt.wiz.biz/", "wss://no.str.cr" relay_manager = RelayManager() - private_key: PrivateKey - public_key: PublicKey - - def __init__(self, privatekey_hex: str = "", relays: List[str] = [], connect=True): - self.generate_keys(privatekey_hex) + def __init__(self, relays: List[str] = [], connect=True): if len(relays): self.relays = relays if connect: @@ -43,87 +29,6 @@ class NostrClient: def close(self): self.relay_manager.close_connections() - def generate_keys(self, privatekey_hex: str = None): - pk = bytes.fromhex(privatekey_hex) if privatekey_hex else None - self.private_key = PrivateKey(pk) - self.public_key = self.private_key.public_key - - def post(self, message: str): - event = Event(message, self.public_key.hex(), kind=EventKind.TEXT_NOTE) - self.private_key.sign_event(event) - event_json = event.to_message() - # print("Publishing message:") - # print(event_json) - self.relay_manager.publish_message(event_json) - - def get_post( - self, sender_publickey: PublicKey = None, callback_func=None, filter_kwargs={} - ): - filter = Filter( - authors=[sender_publickey.hex()] if sender_publickey else None, - kinds=[EventKind.TEXT_NOTE], - **filter_kwargs, - ) - filters = Filters([filter]) - subscription_id = os.urandom(4).hex() - self.relay_manager.add_subscription(subscription_id, filters) - - request = [ClientMessageType.REQUEST, subscription_id] - request.extend(filters.to_json_array()) - message = json.dumps(request) - self.relay_manager.publish_message(message) - - while True: - while self.relay_manager.message_pool.has_events(): - event_msg = self.relay_manager.message_pool.get_event() - if callback_func: - callback_func(event_msg.event) - time.sleep(0.1) - - def dm(self, message: str, to_pubkey: PublicKey): - dm = EncryptedDirectMessage( - recipient_pubkey=to_pubkey.hex(), cleartext_content=message - ) - self.private_key.sign_event(dm) - self.relay_manager.publish_event(dm) - - def get_dm(self, sender_publickey: PublicKey, callback_func=None): - filters = Filters( - [ - Filter( - kinds=[EventKind.ENCRYPTED_DIRECT_MESSAGE], - pubkey_refs=[sender_publickey.hex()], - ) - ] - ) - subscription_id = os.urandom(4).hex() - self.relay_manager.add_subscription(subscription_id, filters) - - request = [ClientMessageType.REQUEST, subscription_id] - request.extend(filters.to_json_array()) - message = json.dumps(request) - self.relay_manager.publish_message(message) - - while True: - while self.relay_manager.message_pool.has_events(): - event_msg = self.relay_manager.message_pool.get_event() - if "?iv=" in event_msg.event.content: - try: - shared_secret = self.private_key.compute_shared_secret( - event_msg.event.public_key - ) - aes = cbc.AESCipher(key=shared_secret) - enc_text_b64, iv_b64 = event_msg.event.content.split("?iv=") - iv = base64.decodebytes(iv_b64.encode("utf-8")) - enc_text = base64.decodebytes(enc_text_b64.encode("utf-8")) - dec_text = aes.decrypt(iv, enc_text) - if callback_func: - callback_func(event_msg.event, dec_text) - except: - pass - break - time.sleep(0.1) - def subscribe( self, callback_events_func=None, diff --git a/nostr/relay.py b/nostr/relay.py index b6207b5..94e532c 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -122,11 +122,6 @@ class Relay: with self.lock: self.subscriptions.pop(id) - def update_subscription(self, id: str, filters: Filters) -> None: - with self.lock: - subscription = self.subscriptions[id] - subscription.filters = filters - def to_json_object(self) -> dict: return { "url": self.url, diff --git a/tasks.py b/tasks.py index eb5391a..069c57d 100644 --- a/tasks.py +++ b/tasks.py @@ -31,7 +31,6 @@ async def subscribe_events(): await asyncio.sleep(2) def callback_events(eventMessage: EventMessage): - # print(f"From {event.public_key[:3]}..{event.public_key[-3:]}: {event.content}") if eventMessage.subscription_id in received_subscription_events: # do not add duplicate events (by event id) if eventMessage.event.id in set( From e2458d43dfdf1028d288810839f45fbc26e462a9 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:00 +0300 Subject: [PATCH 28/81] chore: code clean-up --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index faf5722..0ff85a8 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -11,7 +11,7 @@ class NostrClient: "wss://nostr.zebedee.cloud", "wss://nodestr.fmt.wiz.biz", "wss://nostr.oxtr.dev", - ] # ["wss://nostr.oxtr.dev"] # ["wss://relay.nostr.info"] "wss://nostr-pub.wellorder.net" "ws://91.237.88.218:2700", "wss://nostrrr.bublina.eu.org", ""wss://nostr-relay.freeberty.net"", , "wss://nostr.oxtr.dev", "wss://relay.nostr.info", "wss://nostr-pub.wellorder.net" , "wss://relayer.fiatjaf.com", "wss://nodestr.fmt.wiz.biz/", "wss://no.str.cr" + ] relay_manager = RelayManager() def __init__(self, relays: List[str] = [], connect=True): From 6c66b71302c85198ff890db04ff2af60e9c2adac Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:23 +0300 Subject: [PATCH 29/81] chore: code clean-up --- nostr/relay_manager.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 0ec324a..004b197 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -79,14 +79,3 @@ class RelayManager: for relay in self.relays.values(): if relay.policy.should_write: relay.publish(message) - - def publish_event(self, event: Event): - """Verifies that the Event is publishable before submitting it to relays""" - if event.signature is None: - raise RelayException(f"Could not publish {event.id}: must be signed") - - if not event.verify(): - raise RelayException( - f"Could not publish {event.id}: failed to verify signature {event.signature}" - ) - self.publish_message(event.to_message()) From 62641b56a6c7d0283d938b92cd69b437addbcb65 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:42 +0300 Subject: [PATCH 30/81] chore: code clean-up --- views_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/views_api.py b/views_api.py index 88193ad..26a68e9 100644 --- a/views_api.py +++ b/views_api.py @@ -17,7 +17,6 @@ from .models import Relay, RelayList, TestMessage, TestMessageResponse from .nostr.key import EncryptedDirectMessage, PrivateKey from .nostr.relay import Relay as NostrRelay from .services import NostrRouter, nostr -from .tasks import init_relays # we keep this in all_routers: list[NostrRouter] = [] From 414ae16cb0cbad9d457730a16925d2984e9563a1 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:06:57 +0300 Subject: [PATCH 31/81] refactor: split large methods --- nostr/relay.py | 1 + services.py | 162 ++++++++++++++++++++++++++----------------------- 2 files changed, 86 insertions(+), 77 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 94e532c..8d3545b 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -121,6 +121,7 @@ class Relay: def close_subscription(self, id: str) -> None: with self.lock: self.subscriptions.pop(id) + self.publish(json.dumps(["CLOSE", id])) def to_json_object(self) -> dict: return { diff --git a/services.py b/services.py index b270539..42e8c2c 100644 --- a/services.py +++ b/services.py @@ -2,17 +2,16 @@ import asyncio import json from typing import List, Union -from fastapi import WebSocket, WebSocketDisconnect +from fastapi import WebSocketDisconnect from loguru import logger from lnbits.helpers import urlsafe_short_hash -from .models import Event, Filter, Filters, Relay, RelayList +from .models import Event, Filter from .nostr.client.client import NostrClient as NostrClientLib -from .nostr.event import Event as NostrEvent from .nostr.filter import Filter as NostrFilter from .nostr.filter import Filters as NostrFilters -from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage +from .nostr.message_pool import EndOfStoredEventsMessage, NoticeMessage received_subscription_events: dict[str, list[Event]] = {} received_subscription_notices: list[NoticeMessage] = [] @@ -33,7 +32,7 @@ class NostrRouter: self.connected: bool = True self.websocket = websocket self.tasks: List[asyncio.Task] = [] - self.oridinal_subscription_ids = {} + self.original_subscription_ids = {} async def client_to_nostr(self): """Receives requests / data from the client and forwards it to relays. If the @@ -47,17 +46,7 @@ class NostrRouter: self.connected = False break - # registers a subscription if the input was a REQ request - subscription_id, json_str_rewritten = await self._handle_nostr_subscription( - json_str - ) - - if subscription_id and json_str_rewritten: - self.subscriptions.append(subscription_id) - - # publish data - publish_data = json_str_rewritten or json_str - nostr.client.relay_manager.publish_message(publish_data) + await self._handle_client_to_nostr(json_str) async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -67,50 +56,12 @@ class NostrRouter: that we had previously rewritten in order to avoid collisions when multiple clients use the same id. """ while True and self.connected: - for s in self.subscriptions: - if s in received_subscription_events: - while len(received_subscription_events[s]): - my_event = received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } - - # this reconstructs the original response from the relay - # reconstruct original subscription id - s_original = self.oridinal_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - - # print("Event to forward") - # print(json.dumps(event_to_forward)) - - # send data back to client - await self.websocket.send_text(json.dumps(event_to_forward)) - if s in received_subscription_eosenotices: - my_event = received_subscription_eosenotices[s] - s_original = self.oridinal_subscription_ids[s] - event_to_forward = ["EOSE", s_original] - del received_subscription_eosenotices[s] - # send data back to client - # print("Sending EOSE", event_to_forward) - await self.websocket.send_text(json.dumps(event_to_forward)) - - # if s in received_subscription_notices: - while len(received_subscription_notices): - my_event = received_subscription_notices.pop(0) - event_to_forward = ["NOTICE", my_event.content] - # send data back to client - logger.debug("Nostrclient: Received notice", event_to_forward[1]) - # note: we don't send it to the user because we don't know who should receive it - # await self.websocket.send_text(json.dumps(event_to_forward)) + await self._handle_subscriptions() + self._handle_notices() + await asyncio.sleep(0.1) + async def start(self): self.tasks.append(asyncio.create_task(self.client_to_nostr())) self.tasks.append(asyncio.create_task(self.nostr_to_client())) @@ -120,6 +71,53 @@ class NostrRouter: t.cancel() self.connected = False + async def _handle_subscriptions(self): + for s in self.subscriptions: + if s in received_subscription_events: + await self._handle_received_subscription_events(s) + if s in received_subscription_eosenotices: + await self._handle_received_subscription_eosenotices(s) + + + + async def _handle_received_subscription_eosenotices(self, s): + my_event = received_subscription_eosenotices[s] + s_original = self.original_subscription_ids[s] + event_to_forward = ["EOSE", s_original] + del received_subscription_eosenotices[s] + # send data back to client + # print("Sending EOSE", event_to_forward) + await self.websocket.send_text(json.dumps(event_to_forward)) + + async def _handle_received_subscription_events(self, s): + while len(received_subscription_events[s]): + my_event = received_subscription_events[s].pop(0) + # event.to_message() does not include the subscription ID, we have to add it manually + event_json = { + "id": my_event.id, + "pubkey": my_event.public_key, + "created_at": my_event.created_at, + "kind": my_event.kind, + "tags": my_event.tags, + "content": my_event.content, + "sig": my_event.signature, + } + + # this reconstructs the original response from the relay + # reconstruct original subscription id + s_original = self.original_subscription_ids[s] + event_to_forward = ["EVENT", s_original, event_json] + await self.websocket.send_text(json.dumps(event_to_forward)) + + def _handle_notices(self): + while len(received_subscription_notices): + my_event = received_subscription_notices.pop(0) + event_to_forward = ["NOTICE", my_event.content] + # note: we don't send it to the user because we don't know who should receive it + logger.debug("Nostrclient: Received notice", event_to_forward[1]) + + + def _marshall_nostr_filters(self, data: Union[dict, list]): filters = data if isinstance(data, list) else [data] filters = [Filter.parse_obj(f) for f in filters] @@ -139,7 +137,7 @@ class NostrRouter: ) return NostrFilters(filter_list) - async def _handle_nostr_subscription(self, json_str): + async def _handle_client_to_nostr(self, json_str): """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will register the subscription in the nostr client library that we're using so we can receive the callbacks on it later. Will rewrite the subscription id since we expect @@ -147,25 +145,35 @@ class NostrRouter: """ json_data = json.loads(json_str) assert len(json_data) + if json_data[0] == "REQ": - subscription_id = json_data[1] - subscription_id_rewritten = urlsafe_short_hash() - self.oridinal_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2] - filters = self._marshall_nostr_filters(fltr) - nostr.client.relay_manager.add_subscription( + self._handle_client_req(json_data) + return + + if json_data[0] == "CLOSE": + self.handle_client_close(json_data[1]) + return + + if json_data[0] == "EVENT": + nostr.client.relay_manager.publish_message(json_str) + return + + def _handle_client_req(self, json_data): + subscription_id = json_data[1] + subscription_id_rewritten = urlsafe_short_hash() + self.original_subscription_ids[subscription_id_rewritten] = subscription_id + fltr = json_data[2] + filters = self._marshall_nostr_filters(fltr) + + nostr.client.relay_manager.add_subscription( subscription_id_rewritten, filters ) - request_rewritten = json.dumps( - [json_data[0], subscription_id_rewritten, fltr] - ) - return subscription_id_rewritten, request_rewritten - elif json_data[0] == "CLOSE": - subscription_id = json_data[1] - subscription_id_rewritten = next((k for k, v in self.oridinal_subscription_ids.items() if v == subscription_id), None) - if subscription_id_rewritten: - nostr.client.relay_manager.close_subscription(subscription_id_rewritten) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten]) - return None, request_rewritten + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten, fltr]) + + self.subscriptions.append(subscription_id_rewritten) + nostr.client.relay_manager.publish_message(request_rewritten) - return None, None + def handle_client_close(self, subscription_id): + subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) + if subscription_id_rewritten: + nostr.client.relay_manager.close_subscription(subscription_id_rewritten) From 4238be498f655693604fa1df79fdc6e9eab56a96 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 13:19:22 +0300 Subject: [PATCH 32/81] chore: code clean-up --- nostr/relay.py | 6 ------ nostr/relay_manager.py | 1 - services.py | 6 ++---- tasks.py | 3 --- views_api.py | 1 - 5 files changed, 2 insertions(+), 15 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 8d3545b..cc992f0 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -47,7 +47,6 @@ class Relay: self.queue = Queue() def connect(self, ssl_options: dict = None, proxy: dict = None): - print("### relay.connect", self.url) self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -84,7 +83,6 @@ class Relay: @property def ping(self): - print("### ping: ", self.url) ping_ms = int((self.ws.last_pong_tm - self.ws.last_ping_tm) * 1000) return ping_ms if self.connected and ping_ms > 0 else 0 @@ -98,21 +96,17 @@ class Relay: self.publish(json_str) def queue_worker(self): - print("#### IN !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) while True: if self.connected: try: message = self.queue.get(timeout=1) - print("#### queue_worker", message) self.num_sent_events += 1 self.ws.send(message) except Exception as e: if self.shutdown: - print("#### !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! e [", e, self.url, self.shutdown," ]###") break else: time.sleep(0.1) - print("#### OUT !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!", self.url) def add_subscription(self, id, filters: Filters): with self.lock: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 004b197..b2e4fc2 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -2,7 +2,6 @@ import ssl import threading -from .event import Event from .filter import Filters from .message_pool import MessagePool from .relay import Relay, RelayPolicy diff --git a/services.py b/services.py index 42e8c2c..ed87b39 100644 --- a/services.py +++ b/services.py @@ -81,12 +81,10 @@ class NostrRouter: async def _handle_received_subscription_eosenotices(self, s): - my_event = received_subscription_eosenotices[s] s_original = self.original_subscription_ids[s] event_to_forward = ["EOSE", s_original] del received_subscription_eosenotices[s] - # send data back to client - # print("Sending EOSE", event_to_forward) + await self.websocket.send_text(json.dumps(event_to_forward)) async def _handle_received_subscription_events(self, s): @@ -114,7 +112,7 @@ class NostrRouter: my_event = received_subscription_notices.pop(0) event_to_forward = ["NOTICE", my_event.content] # note: we don't send it to the user because we don't know who should receive it - logger.debug("Nostrclient: Received notice", event_to_forward[1]) + logger.debug("Nostrclient: Received notice: ", event_to_forward[1]) diff --git a/tasks.py b/tasks.py index 069c57d..1db0d86 100644 --- a/tasks.py +++ b/tasks.py @@ -4,10 +4,7 @@ import ssl import threading from .crud import get_relays -from .nostr.event import Event -from .nostr.key import PublicKey from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage -from .nostr.relay_manager import RelayManager from .services import ( nostr, received_subscription_eosenotices, diff --git a/views_api.py b/views_api.py index 26a68e9..a16599b 100644 --- a/views_api.py +++ b/views_api.py @@ -1,5 +1,4 @@ import asyncio -import json from http import HTTPStatus from typing import List, Optional From c0632cabe53aa579cd9753fde8170a3a5bed0cbf Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 14:33:26 +0300 Subject: [PATCH 33/81] refactor: move stop logic --- services.py | 11 ++++++++++- views_api.py | 5 ----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/services.py b/services.py index ed87b39..5546d90 100644 --- a/services.py +++ b/services.py @@ -68,7 +68,16 @@ class NostrRouter: async def stop(self): for t in self.tasks: - t.cancel() + try: + t.cancel() + except: + pass + + for s in self.subscriptions: + try: + nostr.client.relay_manager.close_subscription(s) + except: + pass self.connected = False async def _handle_subscriptions(self): diff --git a/views_api.py b/views_api.py index a16599b..1a8d227 100644 --- a/views_api.py +++ b/views_api.py @@ -152,11 +152,6 @@ async def ws_relay(websocket: WebSocket) -> None: while True: await asyncio.sleep(10) if not router.connected: - for s in router.subscriptions: - try: - nostr.client.relay_manager.close_subscription(s) - except: - pass await router.stop() all_routers.remove(router) break From af14e1c47bb6afc426e711af86673f2eb9965c73 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 15:15:00 +0300 Subject: [PATCH 34/81] fix: do not lose subscriptions if no relay --- nostr/relay_manager.py | 9 +++++++++ views_api.py | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index b2e4fc2..63eb68f 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,4 +1,5 @@ +from asyncio import Lock import ssl import threading @@ -18,6 +19,8 @@ class RelayManager: self.threads: dict[str, threading.Thread] = {} self.queue_threads: dict[str, threading.Thread] = {} self.message_pool = MessagePool() + self._cached_subscriptions = dict[str, Subscription] = {} + self._subscriptions_lock = Lock() def add_relay( self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} @@ -46,10 +49,16 @@ class RelayManager: self.relays.pop(url) def add_subscription(self, id: str, filters: Filters): + with self._subscriptions_lock: + self._cached_subscriptions[id] = Subscription(id, filters) + for relay in self.relays.values(): relay.add_subscription(id, filters) def close_subscription(self, id: str): + with self._subscriptions_lock: + self._cached_subscriptions.pop(id) + for relay in self.relays.values(): relay.close_subscription(id) diff --git a/views_api.py b/views_api.py index 1a8d227..0036070 100644 --- a/views_api.py +++ b/views_api.py @@ -62,11 +62,9 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() if len(all_relays): - subscriptions = all_relays[0].subscriptions nostr.client.relays.append(relay.url) - nostr.client.relay_manager.add_relay(subscriptions) + nostr.client.relay_manager.add_relay() - nostr.client.relay_manager.connect_relay(relay.url) return await get_relays() From 811bfdc45a707192d0c43ac91738ae76344cbc3f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 16:55:09 +0300 Subject: [PATCH 35/81] fix: init new relays with previous subscriptions --- nostr/client/client.py | 4 ++-- nostr/relay_manager.py | 16 ++++++++-------- views_api.py | 6 ++---- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 0ff85a8..7c6063e 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -20,9 +20,9 @@ class NostrClient: if connect: self.connect() - async def connect(self, subscriptions: dict[str, Subscription] = {}): + async def connect(self): for relay in self.relays: - self.relay_manager.add_relay(relay, subscriptions) + self.relay_manager.add_relay(relay) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 63eb68f..ddc833c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,5 +1,4 @@ -from asyncio import Lock import ssl import threading @@ -19,17 +18,18 @@ class RelayManager: self.threads: dict[str, threading.Thread] = {} self.queue_threads: dict[str, threading.Thread] = {} self.message_pool = MessagePool() - self._cached_subscriptions = dict[str, Subscription] = {} - self._subscriptions_lock = Lock() + self._cached_subscriptions: dict[str, Subscription] = {} + self._subscriptions_lock = threading.Lock() - def add_relay( - self, url: str, read: bool = True, write: bool = True, subscriptions: dict[str, Subscription] = {} - ) -> Relay: + def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: if url in self.relays: return - + + with self._subscriptions_lock: + subscriptions = self._cached_subscriptions.copy() + policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions.copy()) + relay = Relay(url, policy, self.message_pool, subscriptions) self.relays[url] = relay self.open_connection( diff --git a/views_api.py b/views_api.py index 0036070..131da44 100644 --- a/views_api.py +++ b/views_api.py @@ -60,10 +60,8 @@ async def api_add_relay(relay: Relay) -> Optional[RelayList]: relay.id = urlsafe_short_hash() await add_relay(relay) - all_relays: List[NostrRelay] = nostr.client.relay_manager.relays.values() - if len(all_relays): - nostr.client.relays.append(relay.url) - nostr.client.relay_manager.add_relay() + nostr.client.relays.append(relay.url) + nostr.client.relay_manager.add_relay(relay.url) return await get_relays() From defb9b8963efe01f4386c472c4c7d7953ddfe09d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 17:10:34 +0300 Subject: [PATCH 36/81] chore: code clean-up --- nostr/client/client.py | 9 +-------- tasks.py | 2 -- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 7c6063e..adc1a6b 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -6,12 +6,7 @@ from ..subscription import Subscription class NostrClient: - relays = [ - "wss://nostr-pub.wellorder.net", - "wss://nostr.zebedee.cloud", - "wss://nodestr.fmt.wiz.biz", - "wss://nostr.oxtr.dev", - ] + relays = [ ] relay_manager = RelayManager() def __init__(self, relays: List[str] = [], connect=True): @@ -24,8 +19,6 @@ class NostrClient: for relay in self.relays: self.relay_manager.add_relay(relay) - - def close(self): self.relay_manager.close_connections() diff --git a/tasks.py b/tasks.py index 1db0d86..7d471fc 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,4 @@ import asyncio -import json -import ssl import threading from .crud import get_relays From 64426d187c31c1ae50f2b020b0d53a5a0e403c09 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 22 Jun 2023 17:26:47 +0300 Subject: [PATCH 37/81] feat: add predefined relay list plus code format --- templates/nostrclient/index.html | 191 +++++++++---------------------- 1 file changed, 53 insertions(+), 138 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 10fd4a5..7e83b6d 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -6,18 +6,18 @@
- +
- Add relay - + + + + + + + +
@@ -29,36 +29,18 @@
Nostrclient
- +
- + @@ -255,6 +256,13 @@ label: 'Ping', field: 'ping' } + , + { + name: 'delete', + align: 'center', + label: '', + field: '' + } ], pagination: { rowsPerPage: 10 @@ -329,8 +337,14 @@ this.relayToAdd = relayUrl await this.addRelay() }, + showDeleteRelayDialog: function (url) { + LNbits.utils + .confirmDialog(' Are you sure you want to remove this relay?') + .onOk(async () => { + this.deleteRelay(url) + }) + }, deleteRelay(url) { - console.log('DELETE RELAY ' + url) LNbits.api .request( 'DELETE', @@ -338,12 +352,14 @@ this.g.user.wallets[0].adminkey, { url: url } ) - .then(function (response) { - if (response.data) { - console.log(response.data) + .then((response) => { + const relayIndex = this.nostrrelayLinks.indexOf(r => r.url === url) + if (relayIndex !== -1) { + this.nostrrelayLinks.splice(relayIndex, 1) } }) - .catch(function (error) { + .catch((error) => { + console.error(error) LNbits.utils.notifyApiError(error) }) }, From ce8b95c2c7a700885ff533be126bd37876ccf8d7 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 10:21:06 +0300 Subject: [PATCH 45/81] feat: restart disconnected relays --- __init__.py | 4 +++- nostr/relay.py | 44 +++++++++++++++++++++--------------------- nostr/relay_manager.py | 37 ++++++++++++++++++++++++----------- tasks.py | 12 ++++++++++++ 4 files changed, 63 insertions(+), 34 deletions(-) diff --git a/__init__.py b/__init__.py index e98d8b8..ec6f4b4 100644 --- a/__init__.py +++ b/__init__.py @@ -36,7 +36,7 @@ def nostr_renderer(): return template_renderer(["lnbits/extensions/nostrclient/templates"]) -from .tasks import init_relays, subscribe_events +from .tasks import check_relays, init_relays, subscribe_events from .views import * # noqa from .views_api import * # noqa @@ -47,3 +47,5 @@ def nostrclient_start(): scheduled_tasks.append(task1) task2 = loop.create_task(catch_everything_and_restart(subscribe_events)) scheduled_tasks.append(task2) + task3 = loop.create_task(catch_everything_and_restart(check_relays)) + scheduled_tasks.append(task3) diff --git a/nostr/relay.py b/nostr/relay.py index cc992f0..6ff16f5 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -3,6 +3,7 @@ import time from queue import Queue from threading import Lock +from loguru import logger from websocket import WebSocketApp from .event import Event @@ -69,17 +70,12 @@ class Relay: def close(self): self.ws.close() + self.connected = False self.shutdown = True - def check_reconnect(self): - try: - self.close() - except: - pass - self.connected = False - if self.reconnect: - time.sleep(self.error_counter**2) - self.connect(self.ssl_options, self.proxy) + @property + def error_threshold_reached(self): + return self.error_threshold and self.error_counter > self.error_threshold @property def ping(self): @@ -104,6 +100,7 @@ class Relay: self.ws.send(message) except Exception as e: if self.shutdown: + logger.warning(f"Closing queue worker for {self.url}") break else: time.sleep(0.1) @@ -127,31 +124,34 @@ class Relay: ], } - def _on_open(self, class_obj): + def _on_open(self, _): + logger.info(f"Connected to relay: '{self.url}'.") self.connected = True - pass + - def _on_close(self, class_obj, status_code, message): - self.connected = False - if self.error_threshold and self.error_counter > self.error_threshold: - pass - else: - self.check_reconnect() - pass + def _on_close(self, _, status_code, message): + logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") + self.close() - def _on_message(self, class_obj, message: str): + + + + def _on_message(self, _, message: str): if self._is_valid_message(message): self.num_received_events += 1 self.message_pool.add_message(message, self.url) + else: + logger.debug(f"Invalid relay message: '{message}'.") - def _on_error(self, class_obj, error): + def _on_error(self, _, error): + logger.warning(f"Relay error: '{str(error)}'") self.connected = False self.error_counter += 1 - def _on_ping(self, class_obj, message): + def _on_ping(self, _*): return - def _on_pong(self, class_obj, message): + def _on_pong(self, _*): return def _is_valid_message(self, message: str) -> bool: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index ddc833c..3ccc2fd 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -2,6 +2,8 @@ import ssl import threading +from loguru import logger + from .filter import Filters from .message_pool import MessagePool from .relay import Relay, RelayPolicy @@ -22,7 +24,7 @@ class RelayManager: self._subscriptions_lock = threading.Lock() def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: - if url in self.relays: + if url in list(self.relays.keys()): return with self._subscriptions_lock: @@ -32,7 +34,7 @@ class RelayManager: relay = Relay(url, policy, self.message_pool, subscriptions) self.relays[url] = relay - self.open_connection( + self._open_connection( relay, {"cert_reqs": ssl.CERT_NONE} ) # NOTE: This disables ssl certificate verification @@ -62,8 +64,23 @@ class RelayManager: for relay in self.relays.values(): relay.close_subscription(id) + def check_and_restart_relays(self): + stopped_relays = [r for r in self.relays.values() if r.shutdown] + for relay in stopped_relays: + logger.info(f"Restarting connection to relay '{relay.url}'") + self._restart_relay(relay) - def open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + + def close_connections(self): + for relay in self.relays.values(): + relay.close() + + def publish_message(self, message: str): + for relay in self.relays.values(): + if relay.policy.should_write: + relay.publish(message) + + def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): self.threads[relay.url] = threading.Thread( target=relay.connect, args=(ssl_options, proxy), @@ -79,11 +96,9 @@ class RelayManager: ) self.queue_threads[relay.url].start() - def close_connections(self): - for relay in self.relays.values(): - relay.close() - - def publish_message(self, message: str): - for relay in self.relays.values(): - if relay.policy.should_write: - relay.publish(message) + def _restart_relay(self, relay: Relay): + if relay.error_threshold_reached: + return + self.remove_relay(relay.url) + new_relay = self.add_relay(relay.url) + new_relay.error_counter = relay.error_counter \ No newline at end of file diff --git a/tasks.py b/tasks.py index 40ca9d9..05057e7 100644 --- a/tasks.py +++ b/tasks.py @@ -1,6 +1,8 @@ import asyncio import threading +from loguru import logger + from . import nostr from .crud import get_relays from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage @@ -17,6 +19,16 @@ async def init_relays(): await nostr.client.connect() +async def check_relays(): + """ Check relays that have been disconnected """ + while True: + try: + await asyncio.sleep(20) + nostr.client.relay_manager.check_and_restart_relays() + except Exception as e: + logger.warning(f"Cannot restart relays: '{str(e)}'.") + + async def subscribe_events(): while not any([r.connected for r in nostr.client.relay_manager.relays.values()]): await asyncio.sleep(2) From dabc26f8a6eab854efe5c3e56e8a6a3fb62612e5 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 10:53:48 +0300 Subject: [PATCH 46/81] fix: unused params --- nostr/relay.py | 4 ++-- nostr/relay_manager.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 6ff16f5..d98a219 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -148,10 +148,10 @@ class Relay: self.connected = False self.error_counter += 1 - def _on_ping(self, _*): + def _on_ping(self, *_): return - def _on_pong(self, _*): + def _on_pong(self, *_): return def _is_valid_message(self, message: str) -> bool: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 3ccc2fd..01889d9 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -67,7 +67,6 @@ class RelayManager: def check_and_restart_relays(self): stopped_relays = [r for r in self.relays.values() if r.shutdown] for relay in stopped_relays: - logger.info(f"Restarting connection to relay '{relay.url}'") self._restart_relay(relay) @@ -99,6 +98,8 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: return + logger.info(f"Restarting connection to relay '{relay.url}'") + self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) new_relay.error_counter = relay.error_counter \ No newline at end of file From f8d578e6aa9f3d3d881cf0fd52d23e18aa553a31 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 12:20:06 +0300 Subject: [PATCH 47/81] feat: improve error handling and reporting --- models.py | 8 ++- nostr/message_type.py | 11 ++++- nostr/relay.py | 84 ++++++++++++++++++++------------ nostr/relay_manager.py | 3 +- templates/nostrclient/index.html | 19 ++++++++ views_api.py | 18 ++++--- 6 files changed, 101 insertions(+), 42 deletions(-) diff --git a/models.py b/models.py index fe12e3b..1006605 100644 --- a/models.py +++ b/models.py @@ -8,12 +8,18 @@ from pydantic import BaseModel, Field from lnbits.helpers import urlsafe_short_hash +class RelayStatus(BaseModel): + num_sent_events: Optional[int] = 0 + num_received_events: Optional[int] = 0 + error_counter: Optional[int] = 0 + error_list: Optional[List] = [] + class Relay(BaseModel): id: Optional[str] = None url: Optional[str] = None connected: Optional[bool] = None connected_string: Optional[str] = None - status: Optional[str] = None + status: Optional[RelayStatus] = None active: Optional[bool] = None ping: Optional[int] = None diff --git a/nostr/message_type.py b/nostr/message_type.py index 3f5206b..d37cdfd 100644 --- a/nostr/message_type.py +++ b/nostr/message_type.py @@ -3,13 +3,20 @@ class ClientMessageType: REQUEST = "REQ" CLOSE = "CLOSE" + class RelayMessageType: EVENT = "EVENT" NOTICE = "NOTICE" END_OF_STORED_EVENTS = "EOSE" + COMMAND_RESULT = "OK" @staticmethod def is_valid(type: str) -> bool: - if type == RelayMessageType.EVENT or type == RelayMessageType.NOTICE or type == RelayMessageType.END_OF_STORED_EVENTS: + if ( + type == RelayMessageType.EVENT + or type == RelayMessageType.NOTICE + or type == RelayMessageType.END_OF_STORED_EVENTS + or type == RelayMessageType.COMMAND_RESULT + ): return True - return False \ No newline at end of file + return False diff --git a/nostr/relay.py b/nostr/relay.py index d98a219..4c989c2 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,6 +2,7 @@ import json import time from queue import Queue from threading import Lock +from typing import List from loguru import logger from websocket import WebSocketApp @@ -39,6 +40,7 @@ class Relay: self.shutdown: bool = False self.error_counter: int = 0 self.error_threshold: int = 100 + self.error_list: List[str] = [] self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 @@ -100,7 +102,7 @@ class Relay: self.ws.send(message) except Exception as e: if self.shutdown: - logger.warning(f"Closing queue worker for {self.url}") + logger.warning(f"Closing queue worker for '{self.url}'.") break else: time.sleep(0.1) @@ -133,18 +135,14 @@ class Relay: logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") self.close() - - - def _on_message(self, _, message: str): if self._is_valid_message(message): self.num_received_events += 1 self.message_pool.add_message(message, self.url) - else: - logger.debug(f"Invalid relay message: '{message}'.") def _on_error(self, _, error): logger.warning(f"Relay error: '{str(error)}'") + self._append_error_message(str(error)) self.connected = False self.error_counter += 1 @@ -161,33 +159,57 @@ class Relay: message_json = json.loads(message) message_type = message_json[0] + if not RelayMessageType.is_valid(message_type): return False + if message_type == RelayMessageType.EVENT: - if not len(message_json) == 3: - return False - - subscription_id = message_json[1] - with self.lock: - if subscription_id not in self.subscriptions: - return False - - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) - if not event.verify(): - return False - - with self.lock: - subscription = self.subscriptions[subscription_id] - - if subscription.filters and not subscription.filters.match(event): - return False + return self._is_valid_event_message(message_json) + + if message_type == RelayMessageType.COMMAND_RESULT: + return self._is_valid_command_result_message(message, message_json) return True + + def _is_valid_event_message(self, message_json): + if not len(message_json) == 3: + return False + + subscription_id = message_json[1] + with self.lock: + if subscription_id not in self.subscriptions: + return False + + e = message_json[2] + event = Event( + e["content"], + e["pubkey"], + e["created_at"], + e["kind"], + e["tags"], + e["sig"], + ) + if not event.verify(): + return False + + with self.lock: + subscription = self.subscriptions[subscription_id] + + if subscription.filters and not subscription.filters.match(event): + return False + + return True + + def _is_valid_command_result_message(self, message, message_json): + if not len(message_json) < 3: + return False + + if message_json[2] != True: + logger.warning(f"Relay '{self.url}' negative command result: '{message}'") + self._append_error_message(message) + return False + + return True + + def _append_error_message(self, message): + self.error_list = ([message] + self.error_list)[:20] \ No newline at end of file diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 01889d9..5838308 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -102,4 +102,5 @@ class RelayManager: self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) - new_relay.error_counter = relay.error_counter \ No newline at end of file + new_relay.error_counter = relay.error_counter + new_relay.error_list = relay.error_list \ No newline at end of file diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index f200973..8018a92 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -51,6 +51,18 @@
+
+
+ ⬆️ + ⬇️ + ⚠️ + + + +
+
+
+
{{ col.value }} @@ -196,6 +208,13 @@ obj._data = _.clone(obj) obj.theTime = obj.time * 60 - (Date.now() / 1000 - obj.timestamp) obj.time = obj.time + 'mins' + obj.status = { + sentEvents: obj.status.num_sent_events, + receveidEvents: obj.status.num_received_events, + errorCount: obj.status.error_counter, + errorList: obj.status.error_list + + } obj.ping = obj.ping + ' ms' diff --git a/views_api.py b/views_api.py index 316ec4d..f918e3c 100644 --- a/views_api.py +++ b/views_api.py @@ -24,19 +24,23 @@ all_routers: list[NostrRouter] = [] async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) for url, r in nostr.client.relay_manager.relays.items(): - status_text = ( - f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" - ) - connected_text = "🟢" if r.connected else "🔴" + # status_text = ( + # f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" + # ) + # connected_text = "🟢" if r.connected else "🔴" relay_id = urlsafe_short_hash() relays.__root__.append( Relay( id=relay_id, url=url, - connected_string=connected_text, - status=status_text, + connected=r.connected, + status={ + "num_sent_events": r.num_sent_events, + "num_received_events": r.num_received_events, + "error_counter": r.error_counter, + "error_list": r.error_list + }, ping=r.ping, - connected=True, active=True, ) ) From d619b965e71e59143331b523482add58deda684f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 26 Jun 2023 12:59:58 +0300 Subject: [PATCH 48/81] fix: UI for connected --- templates/nostrclient/index.html | 36 ++++++++++++++++---------------- views_api.py | 4 ---- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 8018a92..57c3b73 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -50,30 +50,30 @@ @@ -252,10 +252,10 @@ relayTable: { columns: [ { - name: 'connected_string', + name: 'connected', align: 'left', label: '', - field: 'connected_string' + field: 'connected' }, { name: 'relay', diff --git a/views_api.py b/views_api.py index f918e3c..b681f50 100644 --- a/views_api.py +++ b/views_api.py @@ -24,10 +24,6 @@ all_routers: list[NostrRouter] = [] async def api_get_relays() -> RelayList: relays = RelayList(__root__=[]) for url, r in nostr.client.relay_manager.relays.items(): - # status_text = ( - # f"⬆️ {r.num_sent_events} ⬇️ {r.num_received_events} ⚠️ {r.error_counter}" - # ) - # connected_text = "🟢" if r.connected else "🔴" relay_id = urlsafe_short_hash() relays.__root__.append( Relay( From ada5b2a51d4c805854adab568fd939edb0e57572 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Thu, 29 Jun 2023 15:48:28 +0300 Subject: [PATCH 49/81] fix: unsubscribe --- router.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/router.py b/router.py index 3ee9d0a..264d4a4 100644 --- a/router.py +++ b/router.py @@ -39,7 +39,11 @@ class NostrRouter: self.connected = False break - await self._handle_client_to_nostr(json_str) + try: + await self._handle_client_to_nostr(json_str) + except Exception as e: + logger.debug(f"Failed to handle client message: '{str(e)}'.") + async def nostr_to_client(self): """Sends responses from relays back to the client. Polls the subscriptions of this client @@ -49,10 +53,13 @@ class NostrRouter: that we had previously rewritten in order to avoid collisions when multiple clients use the same id. """ while True and self.connected: - await self._handle_subscriptions() - self._handle_notices() - - await asyncio.sleep(0.1) + try: + await self._handle_subscriptions() + self._handle_notices() + await asyncio.sleep(0.1) + except Exception as e: + logger.debug(f"Failed to handle response for client: '{str(e)}'.") + async def start(self): @@ -112,9 +119,8 @@ class NostrRouter: def _handle_notices(self): while len(NostrRouter.received_subscription_notices): my_event = NostrRouter.received_subscription_notices.pop(0) - event_to_forward = ["NOTICE", my_event.content] # note: we don't send it to the user because we don't know who should receive it - logger.debug("Nostrclient: Received notice: ", event_to_forward[1]) + logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") @@ -143,9 +149,11 @@ class NostrRouter: receive the callbacks on it later. Will rewrite the subscription id since we expect multiple clients to use the router and want to avoid subscription id collisions """ + json_data = json.loads(json_str) assert len(json_data) + if json_data[0] == "REQ": self._handle_client_req(json_data) return @@ -176,4 +184,7 @@ class NostrRouter: def _handle_client_close(self, subscription_id): subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) if subscription_id_rewritten: + self.original_subscription_ids.pop(subscription_id_rewritten) nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + else: + logger.debug(f"Failed to unsubscribe from '{subscription_id}.'") From 39c2f881c85c1337f5634106e82db72f73f65bf9 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 10:52:16 +0300 Subject: [PATCH 50/81] feat: revive relay after 24 hours from the last error --- nostr/relay.py | 7 +++++-- nostr/relay_manager.py | 8 +++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 4c989c2..da3496d 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -41,6 +41,8 @@ class Relay: self.error_counter: int = 0 self.error_threshold: int = 100 self.error_list: List[str] = [] + self.notice_list: List[str] = [] + self.last_error_date: int = 0 self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 @@ -77,7 +79,7 @@ class Relay: @property def error_threshold_reached(self): - return self.error_threshold and self.error_counter > self.error_threshold + return self.error_threshold and self.error_counter >= self.error_threshold @property def ping(self): @@ -212,4 +214,5 @@ class Relay: return True def _append_error_message(self, message): - self.error_list = ([message] + self.error_list)[:20] \ No newline at end of file + self.error_list = ([message] + self.error_list)[:20] + self.last_error_date = int(time.time()) \ No newline at end of file diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index 5838308..f8f852c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,6 +1,7 @@ import ssl import threading +import time from loguru import logger @@ -97,7 +98,12 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: - return + time_since_last_error = time.time() - relay.last_error_date + if time_since_last_error < 60 * 60 * 24: # last day + return + relay.error_counter = 0 + relay.error_list = [] + logger.info(f"Restarting connection to relay '{relay.url}'") self.remove_relay(relay.url) From f244f60c562f8747e8b9dba91440e177a962271d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 10:53:39 +0300 Subject: [PATCH 51/81] fix: make it 2 hours --- nostr/relay_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f8f852c..da650d6 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -99,7 +99,7 @@ class RelayManager: def _restart_relay(self, relay: Relay): if relay.error_threshold_reached: time_since_last_error = time.time() - relay.last_error_date - if time_since_last_error < 60 * 60 * 24: # last day + if time_since_last_error < 60 * 60 * 2: # last day return relay.error_counter = 0 relay.error_list = [] From 1601f71b03ffc2b53e6630f00495d83d50d41583 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 11:46:47 +0300 Subject: [PATCH 52/81] feat: show relay NOTICE --- models.py | 1 + nostr/relay.py | 4 +++- nostr/relay_manager.py | 7 +++++- router.py | 1 + templates/nostrclient/index.html | 38 ++++++++++++++++++++++++++------ views_api.py | 3 ++- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/models.py b/models.py index 1006605..88651fc 100644 --- a/models.py +++ b/models.py @@ -13,6 +13,7 @@ class RelayStatus(BaseModel): num_received_events: Optional[int] = 0 error_counter: Optional[int] = 0 error_list: Optional[List] = [] + notice_list: Optional[List] = [] class Relay(BaseModel): id: Optional[str] = None diff --git a/nostr/relay.py b/nostr/relay.py index da3496d..8b081a3 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -128,11 +128,13 @@ class Relay: ], } + def add_notice(self, notice: str): + self.notice_list = ([notice] + self.notice_list)[:20] + def _on_open(self, _): logger.info(f"Connected to relay: '{self.url}'.") self.connected = True - def _on_close(self, _, status_code, message): logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") self.close() diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index da650d6..b2df735 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -6,7 +6,7 @@ import time from loguru import logger from .filter import Filters -from .message_pool import MessagePool +from .message_pool import MessagePool, NoticeMessage from .relay import Relay, RelayPolicy from .subscription import Subscription @@ -80,6 +80,11 @@ class RelayManager: if relay.policy.should_write: relay.publish(message) + def handle_notice(self, notice: NoticeMessage): + relay = next((r for r in self.relays.values() if r.url == notice.url)) + if relay: + relay.add_notice(notice.content) + def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): self.threads[relay.url] = threading.Thread( target=relay.connect, diff --git a/router.py b/router.py index 264d4a4..0982f76 100644 --- a/router.py +++ b/router.py @@ -121,6 +121,7 @@ class NostrRouter: my_event = NostrRouter.received_subscription_notices.pop(0) # note: we don't send it to the user because we don't know who should receive it logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") + nostr.client.relay_manager.handle_notice(my_event) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 57c3b73..82b149e 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -58,12 +58,15 @@
⬆️ ⬇️ - ⚠️ - + + ⚠️ + - -
-
+ + ⓘ + +
+
@@ -198,6 +201,17 @@
+ + + + + +
+ Close +
+
+
+
{% endraw %} {% endblock %} {% block scripts %} {{ window_vars(user) }} @@ -212,8 +226,8 @@ sentEvents: obj.status.num_sent_events, receveidEvents: obj.status.num_received_events, errorCount: obj.status.error_counter, - errorList: obj.status.error_list - + errorList: obj.status.error_list, + noticeList: obj.status.notice_list } obj.ping = obj.ping + ' ms' @@ -226,6 +240,7 @@ 'HH:mm:ss' ) } + console.log('### obj', obj) return obj } @@ -239,6 +254,10 @@ relayToAdd: '', nostrrelayLinks: [], filter: '', + logData: { + show: false, + data: null + }, testData: { show: false, wsConnection: null, @@ -492,6 +511,11 @@ console.warn(error) } }, + showLogDataDialog: function (data = []) { + console.log('### showLogDataDialog', data) + this.logData.data = data.join('\n') + this.logData.show = true + }, exportlnurldeviceCSV: function () { var self = this LNbits.utils.exportCSV(self.relayTable.columns, this.nostrLinks) diff --git a/views_api.py b/views_api.py index b681f50..b6b4527 100644 --- a/views_api.py +++ b/views_api.py @@ -34,7 +34,8 @@ async def api_get_relays() -> RelayList: "num_sent_events": r.num_sent_events, "num_received_events": r.num_received_events, "error_counter": r.error_counter, - "error_list": r.error_list + "error_list": r.error_list, + "notice_list": r.notice_list, }, ping=r.ping, active=True, From 80b86bf00c751cbcca11701bd9c3ab4ac9f2bd94 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 30 Jun 2023 12:05:28 +0300 Subject: [PATCH 53/81] fix: await even if error --- router.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/router.py b/router.py index 0982f76..e85653c 100644 --- a/router.py +++ b/router.py @@ -56,10 +56,9 @@ class NostrRouter: try: await self._handle_subscriptions() self._handle_notices() - await asyncio.sleep(0.1) except Exception as e: logger.debug(f"Failed to handle response for client: '{str(e)}'.") - + await asyncio.sleep(0.1) async def start(self): From 147af04c20505aa42b21acb9903bf4193a7b504c Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 4 Jul 2023 10:02:47 +0300 Subject: [PATCH 54/81] fix: remove unnecessary `async` --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index e033262..66d722c 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -14,7 +14,7 @@ class NostrClient: if connect: self.connect() - async def connect(self): + def connect(self): for relay in self.relays: self.relay_manager.add_relay(relay) From 3b08714a8472e4113110efb4a68095dd2b78cba6 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 4 Jul 2023 11:48:50 +0300 Subject: [PATCH 55/81] fix: async aaaa --- nostr/client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index 66d722c..e033262 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -14,7 +14,7 @@ class NostrClient: if connect: self.connect() - def connect(self): + async def connect(self): for relay in self.relays: self.relay_manager.add_relay(relay) From 403c8f1b05caf595df092cec7b42831a28ddc089 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 5 Jul 2023 09:09:10 +0300 Subject: [PATCH 56/81] fix: thread not dead --- nostr/relay.py | 10 ++++++---- nostr/relay_manager.py | 9 +++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/nostr/relay.py b/nostr/relay.py index 8b081a3..0583bba 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -102,12 +102,14 @@ class Relay: message = self.queue.get(timeout=1) self.num_sent_events += 1 self.ws.send(message) - except Exception as e: - if self.shutdown: - logger.warning(f"Closing queue worker for '{self.url}'.") - break + except: + pass else: time.sleep(0.1) + + if self.shutdown: + logger.warning(f"Closing queue worker for '{self.url}'.") + break def add_subscription(self, id, filters: Filters): with self.lock: diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index b2df735..a551253 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -44,12 +44,13 @@ class RelayManager: return relay def remove_relay(self, url: str): - self.threads[url].join(timeout=1) - self.threads.pop(url) - self.queue_threads[url].join(timeout=1) - self.queue_threads.pop(url) self.relays[url].close() self.relays.pop(url) + self.threads[url].join(timeout=5) + self.threads.pop(url) + self.queue_threads[url].join(timeout=5) + self.queue_threads.pop(url) + def add_subscription(self, id: str, filters: Filters): with self._subscriptions_lock: From 9d9fbc01895ec7b940b8e29a2fba850b5905755b Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 5 Jul 2023 09:09:20 +0300 Subject: [PATCH 57/81] chore: code clean-up --- templates/nostrclient/index.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index 82b149e..a0c5999 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -240,7 +240,6 @@ 'HH:mm:ss' ) } - console.log('### obj', obj) return obj } @@ -512,7 +511,6 @@ } }, showLogDataDialog: function (data = []) { - console.log('### showLogDataDialog', data) this.logData.data = data.join('\n') this.logData.show = true }, From e6624f76bd0ead9d9f1e2259cfe40d3da7e75622 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Tue, 12 Sep 2023 15:06:28 +0300 Subject: [PATCH 58/81] Performance improvements (#19) * fix: increase the wait time for re-connecting to a relay * fix: blocking sleep * fix: remove blocking sleep * fix: allow multiple filters per request --- nostr/client/client.py | 6 +++--- nostr/relay.py | 5 +++-- nostr/relay_manager.py | 17 ++++++++++------- router.py | 4 ++-- tasks.py | 10 ++++++---- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/nostr/client/client.py b/nostr/client/client.py index e033262..db07a06 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,4 +1,4 @@ -import time +import asyncio from typing import List from ..relay_manager import RelayManager @@ -21,7 +21,7 @@ class NostrClient: def close(self): self.relay_manager.close_connections() - def subscribe( + async def subscribe( self, callback_events_func=None, callback_notices_func=None, @@ -41,4 +41,4 @@ class NostrClient: if callback_eosenotices_func: callback_eosenotices_func(event_msg) - time.sleep(0.1) + await asyncio.sleep(0.5) diff --git a/nostr/relay.py b/nostr/relay.py index 0583bba..caacba0 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -1,3 +1,4 @@ +import asyncio import json import time from queue import Queue @@ -95,7 +96,7 @@ class Relay: json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) self.publish(json_str) - def queue_worker(self): + async def queue_worker(self): while True: if self.connected: try: @@ -105,7 +106,7 @@ class Relay: except: pass else: - time.sleep(0.1) + await asyncio.sleep(1) if self.shutdown: logger.warning(f"Closing queue worker for '{self.url}'.") diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index a551253..f639fb0 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,4 +1,5 @@ +import asyncio import ssl import threading import time @@ -95,20 +96,22 @@ class RelayManager: ) self.threads[relay.url].start() + def wrap_async_queue_worker(): + asyncio.run(relay.queue_worker()) + self.queue_threads[relay.url] = threading.Thread( - target=relay.queue_worker, + target=wrap_async_queue_worker, name=f"{relay.url}-queue", daemon=True, ) self.queue_threads[relay.url].start() def _restart_relay(self, relay: Relay): - if relay.error_threshold_reached: - time_since_last_error = time.time() - relay.last_error_date - if time_since_last_error < 60 * 60 * 2: # last day - return - relay.error_counter = 0 - relay.error_list = [] + time_since_last_error = time.time() - relay.last_error_date + + min_wait_time = min(60 * relay.error_counter, 60 * 60 * 24) # try at least once a day + if time_since_last_error < min_wait_time: + return logger.info(f"Restarting connection to relay '{relay.url}'") diff --git a/router.py b/router.py index e85653c..86a8f41 100644 --- a/router.py +++ b/router.py @@ -170,13 +170,13 @@ class NostrRouter: subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.original_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2] + fltr = json_data[2:] filters = self._marshall_nostr_filters(fltr) nostr.client.relay_manager.add_subscription( subscription_id_rewritten, filters ) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten, fltr]) + request_rewritten = json.dumps([json_data[0], subscription_id_rewritten] + fltr) self.subscriptions.append(subscription_id_rewritten) nostr.client.relay_manager.publish_message(request_rewritten) diff --git a/tasks.py b/tasks.py index 05057e7..4c316bc 100644 --- a/tasks.py +++ b/tasks.py @@ -66,13 +66,15 @@ async def subscribe_events(): return - t = threading.Thread( - target=nostr.client.subscribe, - args=( + def wrap_async_subscribe(): + asyncio.run(nostr.client.subscribe( callback_events, callback_notices, callback_eose_notices, - ), + )) + + t = threading.Thread( + target=wrap_async_subscribe, name="Nostr-event-subscription", daemon=True, ) From 6f5e9e34589044225b372eed01594f01a17fc24f Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Fri, 15 Sep 2023 10:21:38 +0300 Subject: [PATCH 59/81] chore: doc formatting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1de61ba..02d12b7 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # Nostrclient - [LNbits](https://github.com/lnbits/lnbits) extension + For more about LNBits extension check [this tutorial](https://github.com/lnbits/lnbits/wiki/LNbits-Extensions) `nostrclient` is an always-on extension that can open multiple connections to nostr relays and act as a multiplexer for other clients: You open a single websocket to `nostrclient` which then sends the data to multiple relays. The responses from these relays are then sent back to the client. ![2023-03-08 18 11 07](https://user-images.githubusercontent.com/93376500/225265727-369f0f8a-196e-41df-a0d1-98b50a0228be.jpg) - ### Troubleshoot -The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. + +The `Test Endpoint` functionality heps the user to check that the `nostrclient` web-socket endpoint works as expected. The LNbits user can DM itself (or a temp account) from `nostrclient` and verify that the messages are sent and received correctly. https://user-images.githubusercontent.com/2951406/236780745-929c33c2-2502-49be-84a3-db02a7aabc0e.mp4 - From e841183c1022eb50646802f4a907febc6ac691f3 Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Mon, 25 Sep 2023 10:13:55 +0300 Subject: [PATCH 60/81] fix: add extra checks (#21) * fix: add extra checks * fix: remove redundant try-catch --- router.py | 56 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/router.py b/router.py index 86a8f41..cc0a380 100644 --- a/router.py +++ b/router.py @@ -89,31 +89,41 @@ class NostrRouter: async def _handle_received_subscription_eosenotices(self, s): - s_original = self.original_subscription_ids[s] - event_to_forward = ["EOSE", s_original] - del NostrRouter.received_subscription_eosenotices[s] - - await self.websocket.send_text(json.dumps(event_to_forward)) + try: + if s not in self.original_subscription_ids: + return + s_original = self.original_subscription_ids[s] + event_to_forward = ["EOSE", s_original] + del NostrRouter.received_subscription_eosenotices[s] + + await self.websocket.send_text(json.dumps(event_to_forward)) + except Exception as e: + logger.debug(e) async def _handle_received_subscription_events(self, s): - while len(NostrRouter.received_subscription_events[s]): - my_event = NostrRouter.received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } + try: + if s not in NostrRouter.received_subscription_events: + return + while len(NostrRouter.received_subscription_events[s]): + my_event = NostrRouter.received_subscription_events[s].pop(0) + # event.to_message() does not include the subscription ID, we have to add it manually + event_json = { + "id": my_event.id, + "pubkey": my_event.public_key, + "created_at": my_event.created_at, + "kind": my_event.kind, + "tags": my_event.tags, + "content": my_event.content, + "sig": my_event.signature, + } - # this reconstructs the original response from the relay - # reconstruct original subscription id - s_original = self.original_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - await self.websocket.send_text(json.dumps(event_to_forward)) + # this reconstructs the original response from the relay + # reconstruct original subscription id + s_original = self.original_subscription_ids[s] + event_to_forward = ["EVENT", s_original, event_json] + await self.websocket.send_text(json.dumps(event_to_forward)) + except Exception as e: + logger.debug(e) def _handle_notices(self): while len(NostrRouter.received_subscription_notices): @@ -121,7 +131,7 @@ class NostrRouter: # note: we don't send it to the user because we don't know who should receive it logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") nostr.client.relay_manager.handle_notice(my_event) - + def _marshall_nostr_filters(self, data: Union[dict, list]): From d202fe305510f2714c9fb4b1ed2c83869c9e9200 Mon Sep 17 00:00:00 2001 From: Tiago Vasconcelos Date: Tue, 26 Sep 2023 13:41:58 +0100 Subject: [PATCH 61/81] allow custom path (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * allow custom path --------- Co-authored-by: dni ⚡ --- __init__.py | 4 +--- config.json | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/__init__.py b/__init__.py index ec6f4b4..7f573e7 100644 --- a/__init__.py +++ b/__init__.py @@ -2,7 +2,6 @@ import asyncio from typing import List from fastapi import APIRouter -from starlette.staticfiles import StaticFiles from lnbits.db import Database from lnbits.helpers import template_renderer @@ -15,7 +14,6 @@ db = Database("ext_nostrclient") nostrclient_static_files = [ { "path": "/nostrclient/static", - "app": StaticFiles(directory="lnbits/extensions/nostrclient/static"), "name": "nostrclient_static", } ] @@ -33,7 +31,7 @@ nostr = NostrClient() def nostr_renderer(): - return template_renderer(["lnbits/extensions/nostrclient/templates"]) + return template_renderer(["nostrclient/templates"]) from .tasks import check_relays, init_relays, subscribe_events diff --git a/config.json b/config.json index ce8ae18..d8b886b 100644 --- a/config.json +++ b/config.json @@ -2,5 +2,6 @@ "name": "Nostr Client", "short_description": "Nostr client for extensions", "tile": "/nostrclient/static/images/nostr-bitcoin.png", - "contributors": ["calle"] + "contributors": ["calle"], + "min_lnbits_version": "0.11.0" } From ab185bd2c426b2115dac7d49dde7118411b78e45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?dni=20=E2=9A=A1?= Date: Tue, 26 Sep 2023 15:39:33 +0200 Subject: [PATCH 62/81] add release workflow (#22) --- .github/workflows/release.yml | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..7ec9b48 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,59 @@ +on: + push: + tags: + - "v[0-9]+.[0-9]+.[0-9]+" + +jobs: + + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Create github release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + tag: ${{ github.ref_name }} + run: | + gh release create "$tag" --generate-notes + + pullrequest: + needs: [release] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.EXT_GITHUB }} + repository: lnbits/lnbits-extensions + path: './lnbits-extensions' + + - name: setup git user + run: | + git config --global user.name "alan" + git config --global user.email "alan@lnbits.com" + + - name: Create pull request in extensions repo + env: + GH_TOKEN: ${{ secrets.EXT_GITHUB }} + repo_name: "${{ github.event.repository.name }}" + tag: "${{ github.ref_name }}" + branch: "update-${{ github.event.repository.name }}-${{ github.ref_name }}" + title: "[UPDATE] ${{ github.event.repository.name }} to ${{ github.ref_name }}" + body: "https://github.com/lnbits/${{ github.event.repository.name }}/releases/${{ github.ref_name }}" + archive: "https://github.com/lnbits/${{ github.event.repository.name }}/archive/refs/tags/${{ github.ref_name }}.zip" + run: | + cd lnbits-extensions + git checkout -b $branch + + # if there is another open PR + git pull origin $branch || echo "branch does not exist" + + sh util.sh update_extension $repo_name $tag + + git add -A + git commit -am "$title" + git push origin $branch + + # check if pr exists before creating it + gh config set pager cat + check=$(gh pr list -H $branch | wc -l) + test $check -ne 0 || gh pr create --title "$title" --body "$body" --repo lnbits/lnbits-extensions From 16ae9d15a143e194deb767c556591307a39f1e5d Mon Sep 17 00:00:00 2001 From: Vlad Stan Date: Wed, 1 Nov 2023 17:46:42 +0200 Subject: [PATCH 63/81] Stabilize (#24) * refactor: clean-up * refactor: extra logs plus try-catch * refactor: do not use bare `except` * refactor: clean-up redundant fields * chore: pass code checks * chore: code format * refactor: code clean-up * fix: refactoring stuff * refactor: remove un-used file * chore: code clean-up * chore: code clean-up * chore: code-format fix * refactor: remove nostr.client wrapper * refactor: code clean-up * chore: code format * refactor: remove `RelayList` class * refactor: extract smaller methods with try-catch * fix: better exception handling * fix: remove redundant filters * fix: simplify event * chore: code format * fix: code check * fix: code check * fix: simplify `REQ` * fix: more clean-ups * refactor: use simpler method * refactor: re-order and rename * fix: stop logic * fix: subscription close before disconnect * chore: play commit --- __init__.py | 11 +- cbc.py | 26 ---- crud.py | 16 +-- migrations.py | 2 +- models.py | 42 ++---- nostr/__init__.py | 0 nostr/bech32.py | 32 +++-- nostr/client/client.py | 57 ++++++-- nostr/delegation.py | 32 ----- nostr/event.py | 3 +- nostr/filter.py | 134 ----------------- nostr/key.py | 14 +- nostr/message_pool.py | 38 ++--- nostr/relay.py | 165 +++++---------------- nostr/relay_manager.py | 109 ++++++++------ nostr/subscription.py | 10 +- router.py | 178 +++++++++-------------- tasks.py | 70 +++++---- templates/nostrclient/index.html | 238 +++++++++++++++++++++++-------- views_api.py | 62 ++++---- 20 files changed, 522 insertions(+), 717 deletions(-) delete mode 100644 cbc.py delete mode 100644 nostr/__init__.py delete mode 100644 nostr/delegation.py delete mode 100644 nostr/filter.py diff --git a/__init__.py b/__init__.py index 7f573e7..d50998b 100644 --- a/__init__.py +++ b/__init__.py @@ -7,7 +7,7 @@ from lnbits.db import Database from lnbits.helpers import template_renderer from lnbits.tasks import catch_everything_and_restart -from .nostr.client.client import NostrClient as NostrClientLib +from .nostr.client.client import NostrClient db = Database("ext_nostrclient") @@ -22,19 +22,14 @@ nostrclient_ext: APIRouter = APIRouter(prefix="/nostrclient", tags=["nostrclient scheduled_tasks: List[asyncio.Task] = [] -class NostrClient: - def __init__(self): - self.client: NostrClientLib = NostrClientLib(connect=False) - - -nostr = NostrClient() +nostr_client = NostrClient() def nostr_renderer(): return template_renderer(["nostrclient/templates"]) -from .tasks import check_relays, init_relays, subscribe_events +from .tasks import check_relays, init_relays, subscribe_events # noqa from .views import * # noqa from .views_api import * # noqa diff --git a/cbc.py b/cbc.py deleted file mode 100644 index 0d9e04f..0000000 --- a/cbc.py +++ /dev/null @@ -1,26 +0,0 @@ -from Cryptodome.Cipher import AES - -BLOCK_SIZE = 16 - - -class AESCipher(object): - """This class is compatible with crypto.createCipheriv('aes-256-cbc')""" - - def __init__(self, key=None): - self.key = key - - def pad(self, data): - length = BLOCK_SIZE - (len(data) % BLOCK_SIZE) - return data + (chr(length) * length).encode() - - def unpad(self, data): - return data[: -(data[-1] if type(data[-1]) == int else ord(data[-1]))] - - def encrypt(self, plain_text): - cipher = AES.new(self.key, AES.MODE_CBC) - b = plain_text.encode("UTF-8") - return cipher.iv, cipher.encrypt(self.pad(b)) - - def decrypt(self, iv, enc_text): - cipher = AES.new(self.key, AES.MODE_CBC, iv=iv) - return self.unpad(cipher.decrypt(enc_text).decode("UTF-8")) diff --git a/crud.py b/crud.py index 780642d..05ca907 100644 --- a/crud.py +++ b/crud.py @@ -1,21 +1,17 @@ -from typing import List, Optional, Union - -import shortuuid - -from lnbits.helpers import urlsafe_short_hash +from typing import List from . import db -from .models import Relay, RelayList +from .models import Relay -async def get_relays() -> RelayList: - row = await db.fetchall("SELECT * FROM nostrclient.relays") - return RelayList(__root__=row) +async def get_relays() -> List[Relay]: + rows = await db.fetchall("SELECT * FROM nostrclient.relays") + return [Relay.from_row(r) for r in rows] async def add_relay(relay: Relay) -> None: await db.execute( - f""" + """ INSERT INTO nostrclient.relays ( id, url, diff --git a/migrations.py b/migrations.py index 5a30e45..73b9ed8 100644 --- a/migrations.py +++ b/migrations.py @@ -3,7 +3,7 @@ async def m001_initial(db): Initial nostrclient table. """ await db.execute( - f""" + """ CREATE TABLE nostrclient.relays ( id TEXT NOT NULL PRIMARY KEY, url TEXT NOT NULL, diff --git a/models.py b/models.py index 88651fc..e08ade3 100644 --- a/models.py +++ b/models.py @@ -1,9 +1,7 @@ -from dataclasses import dataclass -from typing import Dict, List, Optional +from sqlite3 import Row +from typing import List, Optional -from fastapi import Request -from fastapi.param_functions import Query -from pydantic import BaseModel, Field +from pydantic import BaseModel from lnbits.helpers import urlsafe_short_hash @@ -14,7 +12,8 @@ class RelayStatus(BaseModel): error_counter: Optional[int] = 0 error_list: Optional[List] = [] notice_list: Optional[List] = [] - + + class Relay(BaseModel): id: Optional[str] = None url: Optional[str] = None @@ -28,33 +27,9 @@ class Relay(BaseModel): if not self.id: self.id = urlsafe_short_hash() - -class RelayList(BaseModel): - __root__: List[Relay] - - -class Event(BaseModel): - content: str - pubkey: str - created_at: Optional[int] - kind: int - tags: Optional[List[List[str]]] - sig: str - - -class Filter(BaseModel): - ids: Optional[List[str]] - kinds: Optional[List[int]] - authors: Optional[List[str]] - since: Optional[int] - until: Optional[int] - e: Optional[List[str]] = Field(alias="#e") - p: Optional[List[str]] = Field(alias="#p") - limit: Optional[int] - - -class Filters(BaseModel): - __root__: List[Filter] + @classmethod + def from_row(cls, row: Row) -> "Relay": + return cls(**dict(row)) class TestMessage(BaseModel): @@ -62,6 +37,7 @@ class TestMessage(BaseModel): reciever_public_key: str message: str + class TestMessageResponse(BaseModel): private_key: str public_key: str diff --git a/nostr/__init__.py b/nostr/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/nostr/bech32.py b/nostr/bech32.py index 61a92c4..0ae6c80 100644 --- a/nostr/bech32.py +++ b/nostr/bech32.py @@ -26,19 +26,22 @@ from enum import Enum class Encoding(Enum): """Enumeration type to list the various supported encodings.""" + BECH32 = 1 BECH32M = 2 + CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l" -BECH32M_CONST = 0x2bc830a3 +BECH32M_CONST = 0x2BC830A3 + def bech32_polymod(values): """Internal function that computes the Bech32 checksum.""" - generator = [0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3] + generator = [0x3B6A57B2, 0x26508E6D, 0x1EA119FA, 0x3D4233DD, 0x2A1462B3] chk = 1 for value in values: top = chk >> 25 - chk = (chk & 0x1ffffff) << 5 ^ value + chk = (chk & 0x1FFFFFF) << 5 ^ value for i in range(5): chk ^= generator[i] if ((top >> i) & 1) else 0 return chk @@ -58,6 +61,7 @@ def bech32_verify_checksum(hrp, data): return Encoding.BECH32M return None + def bech32_create_checksum(hrp, data, spec): """Compute the checksum values given HRP and data.""" values = bech32_hrp_expand(hrp) + data @@ -69,26 +73,29 @@ def bech32_create_checksum(hrp, data, spec): def bech32_encode(hrp, data, spec): """Compute a Bech32 string given HRP and data values.""" combined = data + bech32_create_checksum(hrp, data, spec) - return hrp + '1' + ''.join([CHARSET[d] for d in combined]) + return hrp + "1" + "".join([CHARSET[d] for d in combined]) + def bech32_decode(bech): """Validate a Bech32/Bech32m string, and determine HRP and data.""" - if ((any(ord(x) < 33 or ord(x) > 126 for x in bech)) or - (bech.lower() != bech and bech.upper() != bech)): + if (any(ord(x) < 33 or ord(x) > 126 for x in bech)) or ( + bech.lower() != bech and bech.upper() != bech + ): return (None, None, None) bech = bech.lower() - pos = bech.rfind('1') + pos = bech.rfind("1") if pos < 1 or pos + 7 > len(bech) or len(bech) > 90: return (None, None, None) - if not all(x in CHARSET for x in bech[pos+1:]): + if not all(x in CHARSET for x in bech[pos + 1 :]): return (None, None, None) hrp = bech[:pos] - data = [CHARSET.find(x) for x in bech[pos+1:]] + data = [CHARSET.find(x) for x in bech[pos + 1 :]] spec = bech32_verify_checksum(hrp, data) if spec is None: return (None, None, None) return (hrp, data[:-6], spec) + def convertbits(data, frombits, tobits, pad=True): """General power-of-2 base conversion.""" acc = 0 @@ -124,7 +131,12 @@ def decode(hrp, addr): return (None, None) if data[0] == 0 and len(decoded) != 20 and len(decoded) != 32: return (None, None) - if data[0] == 0 and spec != Encoding.BECH32 or data[0] != 0 and spec != Encoding.BECH32M: + if ( + data[0] == 0 + and spec != Encoding.BECH32 + or data[0] != 0 + and spec != Encoding.BECH32M + ): return (None, None) return (data[0], decoded) diff --git a/nostr/client/client.py b/nostr/client/client.py index db07a06..4624ff3 100644 --- a/nostr/client/client.py +++ b/nostr/client/client.py @@ -1,25 +1,36 @@ import asyncio -from typing import List + +from loguru import logger from ..relay_manager import RelayManager class NostrClient: - relays = [ ] relay_manager = RelayManager() - def __init__(self, relays: List[str] = [], connect=True): - if len(relays): - self.relays = relays - if connect: - self.connect() + def __init__(self): + self.running = True - async def connect(self): - for relay in self.relays: - self.relay_manager.add_relay(relay) + def connect(self, relays): + for relay in relays: + try: + self.relay_manager.add_relay(relay) + except Exception as e: + logger.debug(e) + self.running = True + + def reconnect(self, relays): + self.relay_manager.remove_relays() + self.connect(relays) def close(self): - self.relay_manager.close_connections() + try: + self.relay_manager.close_all_subscriptions() + self.relay_manager.close_connections() + + self.running = False + except Exception as e: + logger.error(e) async def subscribe( self, @@ -27,18 +38,36 @@ class NostrClient: callback_notices_func=None, callback_eosenotices_func=None, ): - while True: + while self.running: + self._check_events(callback_events_func) + self._check_notices(callback_notices_func) + self._check_eos_notices(callback_eosenotices_func) + + await asyncio.sleep(0.2) + + def _check_events(self, callback_events_func=None): + try: while self.relay_manager.message_pool.has_events(): event_msg = self.relay_manager.message_pool.get_event() if callback_events_func: callback_events_func(event_msg) + except Exception as e: + logger.debug(e) + + def _check_notices(self, callback_notices_func=None): + try: while self.relay_manager.message_pool.has_notices(): event_msg = self.relay_manager.message_pool.get_notice() if callback_notices_func: callback_notices_func(event_msg) + except Exception as e: + logger.debug(e) + + def _check_eos_notices(self, callback_eosenotices_func=None): + try: while self.relay_manager.message_pool.has_eose_notices(): event_msg = self.relay_manager.message_pool.get_eose_notice() if callback_eosenotices_func: callback_eosenotices_func(event_msg) - - await asyncio.sleep(0.5) + except Exception as e: + logger.debug(e) diff --git a/nostr/delegation.py b/nostr/delegation.py deleted file mode 100644 index 94801f5..0000000 --- a/nostr/delegation.py +++ /dev/null @@ -1,32 +0,0 @@ -import time -from dataclasses import dataclass - - -@dataclass -class Delegation: - delegator_pubkey: str - delegatee_pubkey: str - event_kind: int - duration_secs: int = 30*24*60 # default to 30 days - signature: str = None # set in PrivateKey.sign_delegation - - @property - def expires(self) -> int: - return int(time.time()) + self.duration_secs - - @property - def conditions(self) -> str: - return f"kind={self.event_kind}&created_at<{self.expires}" - - @property - def delegation_token(self) -> str: - return f"nostr:delegation:{self.delegatee_pubkey}:{self.conditions}" - - def get_tag(self) -> list[str]: - """ Called by Event """ - return [ - "delegation", - self.delegator_pubkey, - self.conditions, - self.signature, - ] diff --git a/nostr/event.py b/nostr/event.py index 65b187d..a7d4f1d 100644 --- a/nostr/event.py +++ b/nostr/event.py @@ -122,6 +122,7 @@ class EncryptedDirectMessage(Event): def id(self) -> str: if self.content is None: raise Exception( - "EncryptedDirectMessage `id` is undefined until its message is encrypted and stored in the `content` field" + "EncryptedDirectMessage `id` is undefined until its" + + " message is encrypted and stored in the `content` field" ) return super().id diff --git a/nostr/filter.py b/nostr/filter.py deleted file mode 100644 index f119079..0000000 --- a/nostr/filter.py +++ /dev/null @@ -1,134 +0,0 @@ -from collections import UserList -from typing import List - -from .event import Event, EventKind - - -class Filter: - """ - NIP-01 filtering. - - Explicitly supports "#e" and "#p" tag filters via `event_refs` and `pubkey_refs`. - - Arbitrary NIP-12 single-letter tag filters are also supported via `add_arbitrary_tag`. - If a particular single-letter tag gains prominence, explicit support should be - added. For example: - # arbitrary tag - filter.add_arbitrary_tag('t', [hashtags]) - - # promoted to explicit support - Filter(hashtag_refs=[hashtags]) - """ - - def __init__( - self, - event_ids: List[str] = None, - kinds: List[EventKind] = None, - authors: List[str] = None, - since: int = None, - until: int = None, - event_refs: List[ - str - ] = None, # the "#e" attr; list of event ids referenced in an "e" tag - pubkey_refs: List[ - str - ] = None, # The "#p" attr; list of pubkeys referenced in a "p" tag - limit: int = None, - ) -> None: - self.event_ids = event_ids - self.kinds = kinds - self.authors = authors - self.since = since - self.until = until - self.event_refs = event_refs - self.pubkey_refs = pubkey_refs - self.limit = limit - - self.tags = {} - if self.event_refs: - self.add_arbitrary_tag("e", self.event_refs) - if self.pubkey_refs: - self.add_arbitrary_tag("p", self.pubkey_refs) - - def add_arbitrary_tag(self, tag: str, values: list): - """ - Filter on any arbitrary tag with explicit handling for NIP-01 and NIP-12 - single-letter tags. - """ - # NIP-01 'e' and 'p' tags and any NIP-12 single-letter tags must be prefixed with "#" - tag_key = tag if len(tag) > 1 else f"#{tag}" - self.tags[tag_key] = values - - def matches(self, event: Event) -> bool: - if self.event_ids is not None and event.id not in self.event_ids: - return False - if self.kinds is not None and event.kind not in self.kinds: - return False - if self.authors is not None and event.public_key not in self.authors: - return False - if self.since is not None and event.created_at < self.since: - return False - if self.until is not None and event.created_at > self.until: - return False - if (self.event_refs is not None or self.pubkey_refs is not None) and len( - event.tags - ) == 0: - return False - - if self.tags: - e_tag_identifiers = set([e_tag[0] for e_tag in event.tags]) - for f_tag, f_tag_values in self.tags.items(): - # Omit any NIP-01 or NIP-12 "#" chars on single-letter tags - f_tag = f_tag.replace("#", "") - - if f_tag not in e_tag_identifiers: - # Event is missing a tag type that we're looking for - return False - - # Multiple values within f_tag_values are treated as OR search; an Event - # needs to match only one. - # Note: an Event could have multiple entries of the same tag type - # (e.g. a reply to multiple people) so we have to check all of them. - match_found = False - for e_tag in event.tags: - if e_tag[0] == f_tag and e_tag[1] in f_tag_values: - match_found = True - break - if not match_found: - return False - - return True - - def to_json_object(self) -> dict: - res = {} - if self.event_ids is not None: - res["ids"] = self.event_ids - if self.kinds is not None: - res["kinds"] = self.kinds - if self.authors is not None: - res["authors"] = self.authors - if self.since is not None: - res["since"] = self.since - if self.until is not None: - res["until"] = self.until - if self.limit is not None: - res["limit"] = self.limit - if self.tags: - res.update(self.tags) - - return res - - -class Filters(UserList): - def __init__(self, initlist: "list[Filter]" = []) -> None: - super().__init__(initlist) - self.data: "list[Filter]" - - def match(self, event: Event): - for filter in self.data: - if filter.matches(event): - return True - return False - - def to_json_array(self) -> list: - return [filter.to_json_object() for filter in self.data] diff --git a/nostr/key.py b/nostr/key.py index 8089e11..3803650 100644 --- a/nostr/key.py +++ b/nostr/key.py @@ -1,6 +1,5 @@ import base64 import secrets -from hashlib import sha256 import secp256k1 from cffi import FFI @@ -8,7 +7,6 @@ from cryptography.hazmat.primitives import padding from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from . import bech32 -from .delegation import Delegation from .event import EncryptedDirectMessage, Event, EventKind @@ -37,7 +35,7 @@ class PublicKey: class PrivateKey: def __init__(self, raw_secret: bytes = None) -> None: - if not raw_secret is None: + if raw_secret is not None: self.raw_secret = raw_secret else: self.raw_secret = secrets.token_bytes(32) @@ -79,7 +77,10 @@ class PrivateKey: encryptor = cipher.encryptor() encrypted_message = encryptor.update(padded_data) + encryptor.finalize() - return f"{base64.b64encode(encrypted_message).decode()}?iv={base64.b64encode(iv).decode()}" + return ( + f"{base64.b64encode(encrypted_message).decode()}" + + f"?iv={base64.b64encode(iv).decode()}" + ) def encrypt_dm(self, dm: EncryptedDirectMessage) -> None: dm.content = self.encrypt_message( @@ -116,11 +117,6 @@ class PrivateKey: event.public_key = self.public_key.hex() event.signature = self.sign_message_hash(bytes.fromhex(event.id)) - def sign_delegation(self, delegation: Delegation) -> None: - delegation.signature = self.sign_message_hash( - sha256(delegation.delegation_token.encode()).digest() - ) - def __eq__(self, other): return self.raw_secret == other.raw_secret diff --git a/nostr/message_pool.py b/nostr/message_pool.py index 02f7fd4..a3e6c5f 100644 --- a/nostr/message_pool.py +++ b/nostr/message_pool.py @@ -2,13 +2,15 @@ import json from queue import Queue from threading import Lock -from .event import Event from .message_type import RelayMessageType class EventMessage: - def __init__(self, event: Event, subscription_id: str, url: str) -> None: + def __init__( + self, event: str, event_id: str, subscription_id: str, url: str + ) -> None: self.event = event + self.event_id = event_id self.subscription_id = subscription_id self.url = url @@ -59,18 +61,16 @@ class MessagePool: message_type = message_json[0] if message_type == RelayMessageType.EVENT: subscription_id = message_json[1] - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) + event = message_json[2] + if "id" not in event: + return + event_id = event["id"] + with self.lock: - if not f"{subscription_id}_{event.id}" in self._unique_events: - self._accept_event(EventMessage(event, subscription_id, url)) + if f"{subscription_id}_{event_id}" not in self._unique_events: + self._accept_event( + EventMessage(json.dumps(event), event_id, subscription_id, url) + ) elif message_type == RelayMessageType.NOTICE: self.notices.put(NoticeMessage(message_json[1], url)) elif message_type == RelayMessageType.END_OF_STORED_EVENTS: @@ -78,10 +78,12 @@ class MessagePool: def _accept_event(self, event_message: EventMessage): """ - Event uniqueness is considered per `subscription_id`. - The `subscription_id` is rewritten to be unique and it is the same accross relays. - The same event can come from different subscriptions (from the same client or from different ones). - Clients that have joined later should receive older events. + Event uniqueness is considered per `subscription_id`. The `subscription_id` is + rewritten to be unique and it is the same accross relays. The same event can + come from different subscriptions (from the same client or from different ones). + Clients that have joined later should receive older events. """ self.events.put(event_message) - self._unique_events.add(f"{event_message.subscription_id}_{event_message.event.id}") \ No newline at end of file + self._unique_events.add( + f"{event_message.subscription_id}_{event_message.event_id}" + ) diff --git a/nostr/relay.py b/nostr/relay.py index caacba0..b576cfa 100644 --- a/nostr/relay.py +++ b/nostr/relay.py @@ -2,43 +2,23 @@ import asyncio import json import time from queue import Queue -from threading import Lock from typing import List from loguru import logger from websocket import WebSocketApp -from .event import Event -from .filter import Filters from .message_pool import MessagePool -from .message_type import RelayMessageType from .subscription import Subscription -class RelayPolicy: - def __init__(self, should_read: bool = True, should_write: bool = True) -> None: - self.should_read = should_read - self.should_write = should_write - - def to_json_object(self) -> dict[str, bool]: - return {"read": self.should_read, "write": self.should_write} - - class Relay: - def __init__( - self, - url: str, - policy: RelayPolicy, - message_pool: MessagePool, - subscriptions: dict[str, Subscription] = {}, - ) -> None: + def __init__(self, url: str, message_pool: MessagePool) -> None: self.url = url - self.policy = policy self.message_pool = message_pool - self.subscriptions = subscriptions self.connected: bool = False self.reconnect: bool = True self.shutdown: bool = False + self.error_counter: int = 0 self.error_threshold: int = 100 self.error_list: List[str] = [] @@ -47,12 +27,10 @@ class Relay: self.num_received_events: int = 0 self.num_sent_events: int = 0 self.num_subscriptions: int = 0 - self.ssl_options: dict = {} - self.proxy: dict = {} - self.lock = Lock() + self.queue = Queue() - def connect(self, ssl_options: dict = None, proxy: dict = None): + def connect(self): self.ws = WebSocketApp( self.url, on_open=self._on_open, @@ -62,19 +40,14 @@ class Relay: on_ping=self._on_ping, on_pong=self._on_pong, ) - self.ssl_options = ssl_options - self.proxy = proxy if not self.connected: - self.ws.run_forever( - sslopt=ssl_options, - http_proxy_host=None if proxy is None else proxy.get("host"), - http_proxy_port=None if proxy is None else proxy.get("port"), - proxy_type=None if proxy is None else proxy.get("type"), - ping_interval=5, - ) + self.ws.run_forever(ping_interval=10) def close(self): - self.ws.close() + try: + self.ws.close() + except Exception as e: + logger.warning(f"[Relay: {self.url}] Failed to close websocket: {e}") self.connected = False self.shutdown = True @@ -90,10 +63,9 @@ class Relay: def publish(self, message: str): self.queue.put(message) - def publish_subscriptions(self): - for _, subscription in self.subscriptions.items(): - s = subscription.to_json_object() - json_str = json.dumps(["REQ", s["id"], s["filters"][0]]) + def publish_subscriptions(self, subscriptions: List[Subscription] = []): + for s in subscriptions: + json_str = json.dumps(["REQ", s.id] + s.filters) self.publish(json_str) async def queue_worker(self): @@ -103,55 +75,44 @@ class Relay: message = self.queue.get(timeout=1) self.num_sent_events += 1 self.ws.send(message) - except: + except Exception as _: pass else: await asyncio.sleep(1) - - if self.shutdown: - logger.warning(f"Closing queue worker for '{self.url}'.") - break - def add_subscription(self, id, filters: Filters): - with self.lock: - self.subscriptions[id] = Subscription(id, filters) + if self.shutdown: + logger.warning(f"[Relay: {self.url}] Closing queue worker.") + return def close_subscription(self, id: str) -> None: - with self.lock: - self.subscriptions.pop(id) + try: self.publish(json.dumps(["CLOSE", id])) - - def to_json_object(self) -> dict: - return { - "url": self.url, - "policy": self.policy.to_json_object(), - "subscriptions": [ - subscription.to_json_object() - for subscription in self.subscriptions.values() - ], - } + except Exception as e: + logger.debug(f"[Relay: {self.url}] Failed to close subscription: {e}") def add_notice(self, notice: str): - self.notice_list = ([notice] + self.notice_list)[:20] + self.notice_list = [notice] + self.notice_list def _on_open(self, _): - logger.info(f"Connected to relay: '{self.url}'.") + logger.info(f"[Relay: {self.url}] Connected.") self.connected = True - + self.shutdown = False + def _on_close(self, _, status_code, message): - logger.warning(f"Connection to relay {self.url} closed. Status: '{status_code}'. Message: '{message}'.") + logger.warning( + f"[Relay: {self.url}] Connection closed." + + f" Status: '{status_code}'. Message: '{message}'." + ) self.close() def _on_message(self, _, message: str): - if self._is_valid_message(message): - self.num_received_events += 1 - self.message_pool.add_message(message, self.url) + self.num_received_events += 1 + self.message_pool.add_message(message, self.url) def _on_error(self, _, error): - logger.warning(f"Relay error: '{str(error)}'") + logger.warning(f"[Relay: {self.url}] Error: '{str(error)}'") self._append_error_message(str(error)) - self.connected = False - self.error_counter += 1 + self.close() def _on_ping(self, *_): return @@ -159,65 +120,7 @@ class Relay: def _on_pong(self, *_): return - def _is_valid_message(self, message: str) -> bool: - message = message.strip("\n") - if not message or message[0] != "[" or message[-1] != "]": - return False - - message_json = json.loads(message) - message_type = message_json[0] - - if not RelayMessageType.is_valid(message_type): - return False - - if message_type == RelayMessageType.EVENT: - return self._is_valid_event_message(message_json) - - if message_type == RelayMessageType.COMMAND_RESULT: - return self._is_valid_command_result_message(message, message_json) - - return True - - def _is_valid_event_message(self, message_json): - if not len(message_json) == 3: - return False - - subscription_id = message_json[1] - with self.lock: - if subscription_id not in self.subscriptions: - return False - - e = message_json[2] - event = Event( - e["content"], - e["pubkey"], - e["created_at"], - e["kind"], - e["tags"], - e["sig"], - ) - if not event.verify(): - return False - - with self.lock: - subscription = self.subscriptions[subscription_id] - - if subscription.filters and not subscription.filters.match(event): - return False - - return True - - def _is_valid_command_result_message(self, message, message_json): - if not len(message_json) < 3: - return False - - if message_json[2] != True: - logger.warning(f"Relay '{self.url}' negative command result: '{message}'") - self._append_error_message(message) - return False - - return True - def _append_error_message(self, message): - self.error_list = ([message] + self.error_list)[:20] - self.last_error_date = int(time.time()) \ No newline at end of file + self.error_counter += 1 + self.error_list = [message] + self.error_list + self.last_error_date = int(time.time()) diff --git a/nostr/relay_manager.py b/nostr/relay_manager.py index f639fb0..ff7ca9c 100644 --- a/nostr/relay_manager.py +++ b/nostr/relay_manager.py @@ -1,21 +1,15 @@ - import asyncio -import ssl import threading import time +from typing import List from loguru import logger -from .filter import Filters from .message_pool import MessagePool, NoticeMessage -from .relay import Relay, RelayPolicy +from .relay import Relay from .subscription import Subscription -class RelayException(Exception): - pass - - class RelayManager: def __init__(self) -> None: self.relays: dict[str, Relay] = {} @@ -25,72 +19,97 @@ class RelayManager: self._cached_subscriptions: dict[str, Subscription] = {} self._subscriptions_lock = threading.Lock() - def add_relay(self, url: str, read: bool = True, write: bool = True) -> Relay: + def add_relay(self, url: str) -> Relay: if url in list(self.relays.keys()): - return - - with self._subscriptions_lock: - subscriptions = self._cached_subscriptions.copy() + logger.debug(f"Relay '{url}' already present.") + return self.relays[url] - policy = RelayPolicy(read, write) - relay = Relay(url, policy, self.message_pool, subscriptions) + relay = Relay(url, self.message_pool) self.relays[url] = relay - self._open_connection( - relay, - {"cert_reqs": ssl.CERT_NONE} - ) # NOTE: This disables ssl certificate verification + self._open_connection(relay) - relay.publish_subscriptions() + relay.publish_subscriptions(list(self._cached_subscriptions.values())) return relay def remove_relay(self, url: str): - self.relays[url].close() - self.relays.pop(url) - self.threads[url].join(timeout=5) - self.threads.pop(url) - self.queue_threads[url].join(timeout=5) - self.queue_threads.pop(url) - + try: + self.relays[url].close() + except Exception as e: + logger.debug(e) - def add_subscription(self, id: str, filters: Filters): + if url in self.relays: + self.relays.pop(url) + + try: + self.threads[url].join(timeout=5) + except Exception as e: + logger.debug(e) + + if url in self.threads: + self.threads.pop(url) + + try: + self.queue_threads[url].join(timeout=5) + except Exception as e: + logger.debug(e) + + if url in self.queue_threads: + self.queue_threads.pop(url) + + def remove_relays(self): + relay_urls = list(self.relays.keys()) + for url in relay_urls: + self.remove_relay(url) + + def add_subscription(self, id: str, filters: List[str]): + s = Subscription(id, filters) with self._subscriptions_lock: - self._cached_subscriptions[id] = Subscription(id, filters) + self._cached_subscriptions[id] = s for relay in self.relays.values(): - relay.add_subscription(id, filters) + relay.publish_subscriptions([s]) def close_subscription(self, id: str): - with self._subscriptions_lock: - self._cached_subscriptions.pop(id) + try: + with self._subscriptions_lock: + if id in self._cached_subscriptions: + self._cached_subscriptions.pop(id) - for relay in self.relays.values(): - relay.close_subscription(id) + for relay in self.relays.values(): + relay.close_subscription(id) + except Exception as e: + logger.debug(e) + + def close_subscriptions(self, subscriptions: List[str]): + for id in subscriptions: + self.close_subscription(id) + + def close_all_subscriptions(self): + all_subscriptions = list(self._cached_subscriptions.keys()) + self.close_subscriptions(all_subscriptions) def check_and_restart_relays(self): stopped_relays = [r for r in self.relays.values() if r.shutdown] for relay in stopped_relays: self._restart_relay(relay) - def close_connections(self): for relay in self.relays.values(): relay.close() def publish_message(self, message: str): for relay in self.relays.values(): - if relay.policy.should_write: - relay.publish(message) + relay.publish(message) def handle_notice(self, notice: NoticeMessage): relay = next((r for r in self.relays.values() if r.url == notice.url)) if relay: relay.add_notice(notice.content) - def _open_connection(self, relay: Relay, ssl_options: dict = None, proxy: dict = None): + def _open_connection(self, relay: Relay): self.threads[relay.url] = threading.Thread( target=relay.connect, - args=(ssl_options, proxy), name=f"{relay.url}-thread", daemon=True, ) @@ -98,7 +117,7 @@ class RelayManager: def wrap_async_queue_worker(): asyncio.run(relay.queue_worker()) - + self.queue_threads[relay.url] = threading.Thread( target=wrap_async_queue_worker, name=f"{relay.url}-queue", @@ -108,14 +127,16 @@ class RelayManager: def _restart_relay(self, relay: Relay): time_since_last_error = time.time() - relay.last_error_date - - min_wait_time = min(60 * relay.error_counter, 60 * 60 * 24) # try at least once a day + + min_wait_time = min( + 60 * relay.error_counter, 60 * 60 + ) # try at least once an hour if time_since_last_error < min_wait_time: return - + logger.info(f"Restarting connection to relay '{relay.url}'") self.remove_relay(relay.url) new_relay = self.add_relay(relay.url) new_relay.error_counter = relay.error_counter - new_relay.error_list = relay.error_list \ No newline at end of file + new_relay.error_list = relay.error_list diff --git a/nostr/subscription.py b/nostr/subscription.py index 76da0af..a75c1a1 100644 --- a/nostr/subscription.py +++ b/nostr/subscription.py @@ -1,13 +1,7 @@ -from .filter import Filters +from typing import List class Subscription: - def __init__(self, id: str, filters: Filters=None) -> None: + def __init__(self, id: str, filters: List[str] = None) -> None: self.id = id self.filters = filters - - def to_json_object(self): - return { - "id": self.id, - "filters": self.filters.to_json_array() - } diff --git a/router.py b/router.py index cc0a380..e6ccdef 100644 --- a/router.py +++ b/router.py @@ -1,42 +1,61 @@ import asyncio import json -from typing import List, Union +from typing import Dict, List -from fastapi import WebSocketDisconnect +from fastapi import WebSocket, WebSocketDisconnect from loguru import logger from lnbits.helpers import urlsafe_short_hash -from . import nostr -from .models import Event, Filter -from .nostr.filter import Filter as NostrFilter -from .nostr.filter import Filters as NostrFilters -from .nostr.message_pool import EndOfStoredEventsMessage, NoticeMessage +from . import nostr_client +from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage class NostrRouter: - - received_subscription_events: dict[str, list[Event]] = {} + received_subscription_events: dict[str, List[EventMessage]] = {} received_subscription_notices: list[NoticeMessage] = [] received_subscription_eosenotices: dict[str, EndOfStoredEventsMessage] = {} - def __init__(self, websocket): - self.subscriptions: List[str] = [] + def __init__(self, websocket: WebSocket): self.connected: bool = True - self.websocket = websocket + self.websocket: WebSocket = websocket self.tasks: List[asyncio.Task] = [] - self.original_subscription_ids = {} + self.original_subscription_ids: Dict[str, str] = {} - async def client_to_nostr(self): - """Receives requests / data from the client and forwards it to relays. If the - request was a subscription/filter, registers it with the nostr client lib. - Remembers the subscription id so we can send back responses from the relay to this - client in `nostr_to_client`""" - while True: + @property + def subscriptions(self) -> List[str]: + return list(self.original_subscription_ids.keys()) + + def start(self): + self.connected = True + self.tasks.append(asyncio.create_task(self._client_to_nostr())) + self.tasks.append(asyncio.create_task(self._nostr_to_client())) + + async def stop(self): + nostr_client.relay_manager.close_subscriptions(self.subscriptions) + self.connected = False + + for t in self.tasks: + try: + t.cancel() + except Exception as _: + pass + + try: + await self.websocket.close() + except Exception as _: + pass + + async def _client_to_nostr(self): + """ + Receives requests / data from the client and forwards it to relays. + """ + while self.connected: try: json_str = await self.websocket.receive_text() - except WebSocketDisconnect: - self.connected = False + except WebSocketDisconnect as e: + logger.debug(e) + await self.stop() break try: @@ -44,15 +63,9 @@ class NostrRouter: except Exception as e: logger.debug(f"Failed to handle client message: '{str(e)}'.") - - async def nostr_to_client(self): - """Sends responses from relays back to the client. Polls the subscriptions of this client - stored in `my_subscriptions`. Then gets all responses for this subscription id from `received_subscription_events` which - is filled in tasks.py. Takes one response after the other and relays it back to the client. Reconstructs - the reponse manually because the nostr client lib we're using can't do it. Reconstructs the original subscription id - that we had previously rewritten in order to avoid collisions when multiple clients use the same id. - """ - while True and self.connected: + async def _nostr_to_client(self): + """Sends responses from relays back to the client.""" + while self.connected: try: await self._handle_subscriptions() self._handle_notices() @@ -61,24 +74,6 @@ class NostrRouter: await asyncio.sleep(0.1) - async def start(self): - self.tasks.append(asyncio.create_task(self.client_to_nostr())) - self.tasks.append(asyncio.create_task(self.nostr_to_client())) - - async def stop(self): - for t in self.tasks: - try: - t.cancel() - except: - pass - - for s in self.subscriptions: - try: - nostr.client.relay_manager.close_subscription(s) - except: - pass - self.connected = False - async def _handle_subscriptions(self): for s in self.subscriptions: if s in NostrRouter.received_subscription_events: @@ -86,8 +81,6 @@ class NostrRouter: if s in NostrRouter.received_subscription_eosenotices: await self._handle_received_subscription_eosenotices(s) - - async def _handle_received_subscription_eosenotices(self, s): try: if s not in self.original_subscription_ids: @@ -95,7 +88,7 @@ class NostrRouter: s_original = self.original_subscription_ids[s] event_to_forward = ["EOSE", s_original] del NostrRouter.received_subscription_eosenotices[s] - + await self.websocket.send_text(json.dumps(event_to_forward)) except Exception as e: logger.debug(e) @@ -104,97 +97,62 @@ class NostrRouter: try: if s not in NostrRouter.received_subscription_events: return + while len(NostrRouter.received_subscription_events[s]): - my_event = NostrRouter.received_subscription_events[s].pop(0) - # event.to_message() does not include the subscription ID, we have to add it manually - event_json = { - "id": my_event.id, - "pubkey": my_event.public_key, - "created_at": my_event.created_at, - "kind": my_event.kind, - "tags": my_event.tags, - "content": my_event.content, - "sig": my_event.signature, - } + event_message = NostrRouter.received_subscription_events[s].pop(0) + event_json = event_message.event # this reconstructs the original response from the relay # reconstruct original subscription id s_original = self.original_subscription_ids[s] - event_to_forward = ["EVENT", s_original, event_json] - await self.websocket.send_text(json.dumps(event_to_forward)) + event_to_forward = f"""["EVENT", "{s_original}", {event_json}]""" + await self.websocket.send_text(event_to_forward) except Exception as e: - logger.debug(e) + logger.debug(e) # there are 2900 errors here def _handle_notices(self): while len(NostrRouter.received_subscription_notices): my_event = NostrRouter.received_subscription_notices.pop(0) - # note: we don't send it to the user because we don't know who should receive it - logger.info(f"Relay ('{my_event.url}') notice: '{my_event.content}']") - nostr.client.relay_manager.handle_notice(my_event) - - - - def _marshall_nostr_filters(self, data: Union[dict, list]): - filters = data if isinstance(data, list) else [data] - filters = [Filter.parse_obj(f) for f in filters] - filter_list: list[NostrFilter] = [] - for filter in filters: - filter_list.append( - NostrFilter( - event_ids=filter.ids, # type: ignore - kinds=filter.kinds, # type: ignore - authors=filter.authors, # type: ignore - since=filter.since, # type: ignore - until=filter.until, # type: ignore - event_refs=filter.e, # type: ignore - pubkey_refs=filter.p, # type: ignore - limit=filter.limit, # type: ignore - ) - ) - return NostrFilters(filter_list) + logger.info(f"[Relay '{my_event.url}'] Notice: '{my_event.content}']") + # Note: we don't send it to the user because + # we don't know who should receive it + nostr_client.relay_manager.handle_notice(my_event) async def _handle_client_to_nostr(self, json_str): - """Parses a (string) request from a client. If it is a subscription (REQ) or a CLOSE, it will - register the subscription in the nostr client library that we're using so we can - receive the callbacks on it later. Will rewrite the subscription id since we expect - multiple clients to use the router and want to avoid subscription id collisions - """ - json_data = json.loads(json_str) - assert len(json_data) - + assert len(json_data), "Bad JSON array" if json_data[0] == "REQ": self._handle_client_req(json_data) return - + if json_data[0] == "CLOSE": self._handle_client_close(json_data[1]) return if json_data[0] == "EVENT": - nostr.client.relay_manager.publish_message(json_str) + nostr_client.relay_manager.publish_message(json_str) return def _handle_client_req(self, json_data): subscription_id = json_data[1] subscription_id_rewritten = urlsafe_short_hash() self.original_subscription_ids[subscription_id_rewritten] = subscription_id - fltr = json_data[2:] - filters = self._marshall_nostr_filters(fltr) + filters = json_data[2:] - nostr.client.relay_manager.add_subscription( - subscription_id_rewritten, filters - ) - request_rewritten = json.dumps([json_data[0], subscription_id_rewritten] + fltr) - - self.subscriptions.append(subscription_id_rewritten) - nostr.client.relay_manager.publish_message(request_rewritten) + nostr_client.relay_manager.add_subscription(subscription_id_rewritten, filters) def _handle_client_close(self, subscription_id): - subscription_id_rewritten = next((k for k, v in self.original_subscription_ids.items() if v == subscription_id), None) + subscription_id_rewritten = next( + ( + k + for k, v in self.original_subscription_ids.items() + if v == subscription_id + ), + None, + ) if subscription_id_rewritten: self.original_subscription_ids.pop(subscription_id_rewritten) - nostr.client.relay_manager.close_subscription(subscription_id_rewritten) + nostr_client.relay_manager.close_subscription(subscription_id_rewritten) else: logger.debug(f"Failed to unsubscribe from '{subscription_id}.'") diff --git a/tasks.py b/tasks.py index 4c316bc..69aa33c 100644 --- a/tasks.py +++ b/tasks.py @@ -3,75 +3,69 @@ import threading from loguru import logger -from . import nostr +from . import nostr_client from .crud import get_relays from .nostr.message_pool import EndOfStoredEventsMessage, EventMessage, NoticeMessage -from .router import NostrRouter, nostr +from .router import NostrRouter async def init_relays(): - # reinitialize the entire client - nostr.__init__() # get relays from db relays = await get_relays() # set relays and connect to them - nostr.client.relays = list(set([r.url for r in relays.__root__ if r.url])) - await nostr.client.connect() + valid_relays = list(set([r.url for r in relays if r.url])) + + nostr_client.reconnect(valid_relays) async def check_relays(): - """ Check relays that have been disconnected """ + """Check relays that have been disconnected""" while True: try: await asyncio.sleep(20) - nostr.client.relay_manager.check_and_restart_relays() + nostr_client.relay_manager.check_and_restart_relays() except Exception as e: logger.warning(f"Cannot restart relays: '{str(e)}'.") - + async def subscribe_events(): - while not any([r.connected for r in nostr.client.relay_manager.relays.values()]): + while not any([r.connected for r in nostr_client.relay_manager.relays.values()]): await asyncio.sleep(2) def callback_events(eventMessage: EventMessage): - if eventMessage.subscription_id in NostrRouter.received_subscription_events: - # do not add duplicate events (by event id) - if eventMessage.event.id in set( - [ - e.id - for e in NostrRouter.received_subscription_events[eventMessage.subscription_id] - ] - ): - return + sub_id = eventMessage.subscription_id + if sub_id not in NostrRouter.received_subscription_events: + NostrRouter.received_subscription_events[sub_id] = [eventMessage] + return - NostrRouter.received_subscription_events[eventMessage.subscription_id].append( - eventMessage.event - ) - else: - NostrRouter.received_subscription_events[eventMessage.subscription_id] = [ - eventMessage.event - ] - return + # do not add duplicate events (by event id) + ids = set( + [e.event_id for e in NostrRouter.received_subscription_events[sub_id]] + ) + if eventMessage.event_id in ids: + return + + NostrRouter.received_subscription_events[sub_id].append(eventMessage) def callback_notices(noticeMessage: NoticeMessage): if noticeMessage not in NostrRouter.received_subscription_notices: NostrRouter.received_subscription_notices.append(noticeMessage) - return def callback_eose_notices(eventMessage: EndOfStoredEventsMessage): - if eventMessage.subscription_id not in NostrRouter.received_subscription_eosenotices: - NostrRouter.received_subscription_eosenotices[ - eventMessage.subscription_id - ] = eventMessage + sub_id = eventMessage.subscription_id + if sub_id in NostrRouter.received_subscription_eosenotices: + return - return + NostrRouter.received_subscription_eosenotices[sub_id] = eventMessage def wrap_async_subscribe(): - asyncio.run(nostr.client.subscribe( - callback_events, - callback_notices, - callback_eose_notices, - )) + asyncio.run( + nostr_client.subscribe( + callback_events, + callback_notices, + callback_eose_notices, + ) + ) t = threading.Thread( target=wrap_async_subscribe, diff --git a/templates/nostrclient/index.html b/templates/nostrclient/index.html index a0c5999..db0f98e 100644 --- a/templates/nostrclient/index.html +++ b/templates/nostrclient/index.html @@ -6,13 +6,30 @@
- +
- - - + + @@ -29,18 +46,36 @@
Nostrclient
- +
- +