From 541b19272cf91b57614725759c2149a1295219bd Mon Sep 17 00:00:00 2001 From: boufni95 Date: Thu, 2 Oct 2025 20:12:36 +0000 Subject: [PATCH] users cleanup --- datasource.js | 9 ++-- src/nostrMiddleware.ts | 1 + src/services/main/appUserManager.ts | 53 +++++++++++++++++++ src/services/main/init.ts | 1 + src/services/storage/applicationStorage.ts | 12 ++++- src/services/storage/db/db.ts | 4 +- src/services/storage/debitStorage.ts | 4 ++ src/services/storage/entity/UserAccess.ts | 14 +++++ src/services/storage/managementStorage.ts | 6 ++- .../migrations/1759426050669-user_access.ts | 14 +++++ src/services/storage/migrations/runner.ts | 4 +- src/services/storage/offerStorage.ts | 5 ++ src/services/storage/paymentStorage.ts | 8 +++ src/services/storage/productStorage.ts | 4 ++ src/services/storage/userStorage.ts | 13 +++++ 15 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 src/services/storage/entity/UserAccess.ts create mode 100644 src/services/storage/migrations/1759426050669-user_access.ts diff --git a/datasource.js b/datasource.js index 3d78664c..1fbf411a 100644 --- a/datasource.js +++ b/datasource.js @@ -19,6 +19,7 @@ import { DebitAccess } from "./build/src/services/storage/entity/DebitAccess.js" import { UserOffer } from "./build/src/services/storage/entity/UserOffer.js" import { ManagementGrant } from "./build/src/services/storage/entity/ManagementGrant.js" import { AppUserDevice } from "./build/src/services/storage/entity/AppUserDevice.js" +import { UserAccess } from "./build/src/services/storage/entity/UserAccess.js" import { Initial1703170309875 } from './build/src/services/storage/migrations/1703170309875-initial.js' import { LspOrder1718387847693 } from './build/src/services/storage/migrations/1718387847693-lsp_order.js' @@ -35,6 +36,7 @@ import { ManagementGrant1751307732346 } from './build/src/services/storage/migra import { InvoiceCallbackUrls1752425992291 } from './build/src/services/storage/migrations/1752425992291-invoice_callback_urls.js' import { OldSomethingLeftover1753106599604 } from './build/src/services/storage/migrations/1753106599604-old_something_leftover.js' import { UserReceivingInvoiceIdx1753109184611 } from './build/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.js' +import { AppUserDevice1753285173175 } from './build/src/services/storage/migrations/1753285173175-app_user_device.js' export default new DataSource({ type: "better-sqlite3", @@ -42,10 +44,11 @@ export default new DataSource({ // logging: true, migrations: [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, DebitToPub1727105758354, UserCbUrl1727112281043, - UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611], + UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611, + AppUserDevice1753285173175], entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment, UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder, LndNodeInfo, - TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant, AppUserDevice], + TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant, AppUserDevice, UserAccess], // synchronize: true, }) -//npx typeorm migration:generate ./src/services/storage/migrations/app_user_device -d ./datasource.js \ No newline at end of file +//npx typeorm migration:generate ./src/services/storage/migrations/user_access -d ./datasource.js \ No newline at end of file diff --git a/src/nostrMiddleware.ts b/src/nostrMiddleware.ts index e2fa3b6d..414a73b9 100644 --- a/src/nostrMiddleware.ts +++ b/src/nostrMiddleware.ts @@ -12,6 +12,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett NostrUserAuthGuard: async (appId, pub) => { const app = await mainHandler.storage.applicationStorage.GetApplication(appId || "") const nostrUser = await mainHandler.storage.applicationStorage.GetOrCreateNostrAppUser(app, pub || "") + await mainHandler.storage.userStorage.UpsertUserAccess(nostrUser.user.user_id, Math.floor(Date.now() / 1000)) return { user_id: nostrUser.user.user_id, app_user_id: nostrUser.identifier, app_id: appId || "" } }, NostrAdminAuthGuard: async (appId, pub) => { diff --git a/src/services/main/appUserManager.ts b/src/services/main/appUserManager.ts index 751176ea..fa457a93 100644 --- a/src/services/main/appUserManager.ts +++ b/src/services/main/appUserManager.ts @@ -5,11 +5,13 @@ import * as Types from '../../../proto/autogenerated/ts/types.js' import { MainSettings } from './settings.js' import ApplicationManager from './applicationManager.js' import { OfferPriceType, ndebitEncode, nmanageEncode, nofferEncode } from '@shocknet/clink-sdk' +import { getLogger } from '../helpers/logger.js' export default class { storage: Storage settings: MainSettings applicationManager: ApplicationManager + log = getLogger({ component: 'AppUserManager' }) constructor(storage: Storage, settings: MainSettings, applicationManager: ApplicationManager) { this.storage = storage this.settings = settings @@ -30,6 +32,7 @@ export default class { if (!decoded.user_id || !decoded.app_id || !decoded.app_user_id) { throw new Error("the provided token is not a valid app user token token") } + this.storage.userStorage.UpsertUserAccess(decoded.user_id, Math.floor(Date.now() / 1000)) return decoded } @@ -117,4 +120,54 @@ export default class { const user = await this.storage.applicationStorage.GetApplicationUser(app, ctx.app_user_id); await this.storage.applicationStorage.UpdateAppUserMessagingToken(user.identifier, req.device_id, req.firebase_messaging_token); } + + async CleanupInactiveUsers() { + this.log("Cleaning up inactive users") + const inactiveUsers = await this.storage.userStorage.GetInactiveUsers(30) + const toDelete:{userId: string, appUserIds: string[]}[] = [] + for (const u of inactiveUsers) { + const user = await this.storage.userStorage.GetUser(u.user_id) + if (user.balance_sats > 0) { + continue + } + const txs = await this.storage.paymentStorage.GetUserReceivingTransactions(u.user_id, 0, 1) + if (txs.length > 0) { + continue + } + const invoices = await this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(user.serial_id, 0, 0, 1) + if (invoices.length > 0) { + continue + } + const userToUser = await this.storage.paymentStorage.GetUserToUserReceivedPayments(u.user_id, 0, 1) + if (userToUser.length > 0) { + continue + } + const appUsers = await this.storage.applicationStorage.GetAllAppUsersFromUser(u.user_id) + toDelete.push({userId: u.user_id, appUserIds: appUsers.map(a => a.identifier)}) + } + + this.log("Found",toDelete.length, "inactive users to delete") + // await this.RemoveIntactiveUsers(toDelete) TODO: activate deletion + } + + async RemoveIntactiveUsers(toDelete: { userId: string, appUserIds: string[] }[]) { + this.log("Deleting",toDelete.length, "inactive users") + for (let i = 0; i < toDelete.length; i++) { + const {userId,appUserIds} = toDelete[i] + this.log("Deleting user", userId, "progress", i+1, "/", toDelete.length) + await this.storage.StartTransaction(async tx => { + for (const appUserId of appUserIds) { + await this.storage.managementStorage.removeUserGrants(appUserId, tx) + await this.storage.offerStorage.DeleteUserOffers(appUserId, tx) + await this.storage.debitStorage.RemoveUserDebitAccess(appUserId, tx) + await this.storage.applicationStorage.RemoveAppUserDevices(appUserId, tx) + + } + await this.storage.paymentStorage.RemoveUserInvoices(userId, tx) + await this.storage.productStorage.RemoveUserProducts(userId, tx) + await this.storage.paymentStorage.RemoveUserEphemeralKeys(userId, tx) + }) + } + this.log("Cleaned up inactive users") + } } \ No newline at end of file diff --git a/src/services/main/init.ts b/src/services/main/init.ts index 1f754714..e21eab53 100644 --- a/src/services/main/init.ts +++ b/src/services/main/init.ts @@ -75,6 +75,7 @@ export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings return } await mainHandler.paymentManager.checkPendingPayments() + await mainHandler.appUserManager.CleanupInactiveUsers() await mainHandler.paymentManager.watchDog.Start() return { mainHandler, apps, liquidityProviderInfo, liquidityProviderApp, wizard, adminManager } } diff --git a/src/services/storage/applicationStorage.ts b/src/services/storage/applicationStorage.ts index b00b5f81..21f6f9c9 100644 --- a/src/services/storage/applicationStorage.ts +++ b/src/services/storage/applicationStorage.ts @@ -1,5 +1,5 @@ import crypto from 'crypto'; -import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThanOrEqual } from "typeorm" +import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThanOrEqual, In } from "typeorm" import { generateSecretKey, getPublicKey } from 'nostr-tools'; import { Application } from "./entity/Application.js" import UserStorage from './userStorage.js'; @@ -160,6 +160,12 @@ export default class { this.dbs.Remove('User', baseUser, txId) } + async RemoveAppUsersAndBaseUsers(appUserIds: string[],baseUser:string, txId?: string) { + await this.dbs.Delete('ApplicationUser', { identifier: In(appUserIds) }, txId) + await this.dbs.Delete('User', { user_id: baseUser }, txId) + + } + async AddInviteToken(app: Application, sats?: number) { return this.dbs.CreateAndSave('InviteToken', { @@ -198,4 +204,8 @@ export default class { async GetAppUserDevices(appUserId: string, txId?: string) { return this.dbs.Find('AppUserDevice', { where: { app_user_id: appUserId } }, txId) } + + async RemoveAppUserDevices(appUserId: string, txId?: string) { + return this.dbs.Delete('AppUserDevice', { app_user_id: appUserId }, txId) + } } \ No newline at end of file diff --git a/src/services/storage/db/db.ts b/src/services/storage/db/db.ts index 3c2a954a..881f9124 100644 --- a/src/services/storage/db/db.ts +++ b/src/services/storage/db/db.ts @@ -28,6 +28,7 @@ import { ManagementGrant } from "../entity/ManagementGrant.js" import { ChannelEvent } from "../entity/ChannelEvent.js" import { AppUserDevice } from "../entity/AppUserDevice.js" import * as fs from 'fs' +import { UserAccess } from "../entity/UserAccess.js" export type DbSettings = { @@ -71,7 +72,8 @@ export const MainDbEntities = { 'UserOffer': UserOffer, 'Product': Product, 'ManagementGrant': ManagementGrant, - 'AppUserDevice': AppUserDevice + 'AppUserDevice': AppUserDevice, + 'UserAccess': UserAccess } export type MainDbNames = keyof typeof MainDbEntities export const MainDbEntitiesNames = Object.keys(MainDbEntities) diff --git a/src/services/storage/debitStorage.ts b/src/services/storage/debitStorage.ts index f1ed7357..08ae3133 100644 --- a/src/services/storage/debitStorage.ts +++ b/src/services/storage/debitStorage.ts @@ -50,4 +50,8 @@ export default class { async RemoveDebitAccess(appUserId: string, authorizedPub: string, txId?: string) { return this.dbs.Delete('DebitAccess', { app_user_id: appUserId, npub: authorizedPub }, txId) } + + async RemoveUserDebitAccess(appUserId: string, txId?: string) { + return this.dbs.Delete('DebitAccess', { app_user_id: appUserId }, txId) + } } \ No newline at end of file diff --git a/src/services/storage/entity/UserAccess.ts b/src/services/storage/entity/UserAccess.ts new file mode 100644 index 00000000..266e753d --- /dev/null +++ b/src/services/storage/entity/UserAccess.ts @@ -0,0 +1,14 @@ +import { Entity, PrimaryGeneratedColumn, Column, ManyToOne, CreateDateColumn, UpdateDateColumn, PrimaryColumn } from "typeorm" +import { User } from "./User.js" + +@Entity() +export class UserAccess { + @PrimaryColumn() + user_id: string + + @Column({ default: 0 }) + last_seen_at_unix: number + + @Column({ default: false }) + locked: boolean +} diff --git a/src/services/storage/managementStorage.ts b/src/services/storage/managementStorage.ts index 1cc263b5..20f14311 100644 --- a/src/services/storage/managementStorage.ts +++ b/src/services/storage/managementStorage.ts @@ -22,4 +22,8 @@ export class ManagementStorage { async removeGrant(appUserId: string, appPubkey: string) { return this.dbs.Delete('ManagementGrant', { app_pubkey: appPubkey, app_user_id: appUserId }); } -} \ No newline at end of file + + async removeUserGrants(appUserId: string, txId?: string) { + return this.dbs.Delete('ManagementGrant', { app_user_id: appUserId }, txId); + } +} \ No newline at end of file diff --git a/src/services/storage/migrations/1759426050669-user_access.ts b/src/services/storage/migrations/1759426050669-user_access.ts new file mode 100644 index 00000000..34c8b83a --- /dev/null +++ b/src/services/storage/migrations/1759426050669-user_access.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class UserAccess1759426050669 implements MigrationInterface { + name = 'UserAccess1759426050669' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "user_access" ("user_id" varchar PRIMARY KEY NOT NULL, "last_seen_at_unix" integer NOT NULL DEFAULT (0), "locked" boolean NOT NULL DEFAULT (0))`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TABLE "user_access"`); + } + +} diff --git a/src/services/storage/migrations/runner.ts b/src/services/storage/migrations/runner.ts index 84cd6aa1..59b8ab7e 100644 --- a/src/services/storage/migrations/runner.ts +++ b/src/services/storage/migrations/runner.ts @@ -23,11 +23,13 @@ import { InvoiceCallbackUrls1752425992291 } from './1752425992291-invoice_callba import { AppUserDevice1753285173175 } from './1753285173175-app_user_device.js' import { OldSomethingLeftover1753106599604 } from './1753106599604-old_something_leftover.js' import { UserReceivingInvoiceIdx1753109184611 } from './1753109184611-user_receiving_invoice_idx.js' +import { UserAccess1759426050669 } from './1759426050669-user_access.js' export const allMigrations = [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, TrackedProvider1720814323679, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, - DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611, AppUserDevice1753285173175] + DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, + InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611, AppUserDevice1753285173175, UserAccess1759426050669] export const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538, HtlcCount1724266887195, BalanceEvents1724860966825, RootOps1732566440447, RootOpsTime1745428134124, ChannelEvents1750777346411] /* export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise => { diff --git a/src/services/storage/offerStorage.ts b/src/services/storage/offerStorage.ts index c7942452..f4fbaaca 100644 --- a/src/services/storage/offerStorage.ts +++ b/src/services/storage/offerStorage.ts @@ -27,6 +27,11 @@ export default class { async DeleteUserOffer(appUserId: string, offerId: string, txId?: string) { await this.dbs.Delete('UserOffer', { app_user_id: appUserId, offer_id: offerId }, txId) } + + async DeleteUserOffers(appUserId: string, txId?: string) { + await this.dbs.Delete('UserOffer', { app_user_id: appUserId }, txId) + } + async UpdateUserOffer(app_user_id: string, offerId: string, req: Partial, txId?: string) { return this.dbs.Update('UserOffer', { app_user_id, offer_id: offerId }, req, txId) } diff --git a/src/services/storage/paymentStorage.ts b/src/services/storage/paymentStorage.ts index a77c39a5..cc75de8a 100644 --- a/src/services/storage/paymentStorage.ts +++ b/src/services/storage/paymentStorage.ts @@ -129,6 +129,10 @@ export default class { }, txId) } + async RemoveUserInvoices(userId: string, txId?: string) { + return this.dbs.Delete('UserReceivingInvoice', { user: { user_id: userId } }, txId) + } + async GetAddressOwner(address: string, txId?: string): Promise { return this.dbs.FindOne('UserReceivingAddress', { where: { address } }, txId) } @@ -303,6 +307,10 @@ export default class { return found } + async RemoveUserEphemeralKeys(userId: string, txId?: string) { + return this.dbs.Delete('UserEphemeralKey', { user: { user_id: userId } }, txId) + } + async AddPendingUserToUserPayment(fromUserId: string, toUserId: string, amount: number, fee: number, linkedApplication: Application, txId: string) { return this.dbs.CreateAndSave('UserToUserPayment', { from_user: await this.userStorage.GetUser(fromUserId, txId), diff --git a/src/services/storage/productStorage.ts b/src/services/storage/productStorage.ts index 26c386c5..e6f2f662 100644 --- a/src/services/storage/productStorage.ts +++ b/src/services/storage/productStorage.ts @@ -19,4 +19,8 @@ export default class { } return product } + + async RemoveUserProducts(userId: string, txId?: string) { + return this.dbs.Delete('Product', { owner: { user_id: userId } }, txId) + } } \ No newline at end of file diff --git a/src/services/storage/userStorage.ts b/src/services/storage/userStorage.ts index f4ed6fda..58dd6cf5 100644 --- a/src/services/storage/userStorage.ts +++ b/src/services/storage/userStorage.ts @@ -4,6 +4,8 @@ import { UserBasicAuth } from './entity/UserBasicAuth.js'; import { getLogger } from '../helpers/logger.js'; import EventsLogManager from './eventsLog.js'; import { StorageInterface } from './db/storageInterface.js'; +import { UserAccess } from './entity/UserAccess.js'; +import { LessThan, MoreThan } from 'typeorm'; export default class { dbs: StorageInterface eventsLog: EventsLogManager @@ -113,4 +115,15 @@ export default class { const user = await this.GetUser(userId, txId) await this.dbs.Update('User', user.serial_id, update, txId) } + + async UpsertUserAccess(userId: string, lastSeenAtUnix: number, txId?: string) { + return this.dbs.CreateAndSave('UserAccess', { user_id: userId, last_seen_at_unix: lastSeenAtUnix }, txId) + } + + async GetInactiveUsers(inactiveForDays: number) { + const seconds = inactiveForDays * 24 * 60 * 60 + const now = Math.floor(Date.now() / 1000) + const lastSeenAtUnix = now - seconds + return this.dbs.Find('UserAccess', { where: { last_seen_at_unix: LessThan(lastSeenAtUnix) } }) + } } \ No newline at end of file