Merge pull request #843 from shocknet/users-cleanup

users cleanup

datasource.js

@@ -19,6 +19,7 @@ import { DebitAccess } from "./build/src/services/storage/entity/DebitAccess.js"
 import { UserOffer } from "./build/src/services/storage/entity/UserOffer.js"
 import { ManagementGrant } from "./build/src/services/storage/entity/ManagementGrant.js"
 import { AppUserDevice } from "./build/src/services/storage/entity/AppUserDevice.js"
+import { UserAccess } from "./build/src/services/storage/entity/UserAccess.js"
 
 import { Initial1703170309875 } from './build/src/services/storage/migrations/1703170309875-initial.js'
 import { LspOrder1718387847693 } from './build/src/services/storage/migrations/1718387847693-lsp_order.js'
@@ -35,6 +36,7 @@ import { ManagementGrant1751307732346 } from './build/src/services/storage/migra
 import { InvoiceCallbackUrls1752425992291 } from './build/src/services/storage/migrations/1752425992291-invoice_callback_urls.js'
 import { OldSomethingLeftover1753106599604 } from './build/src/services/storage/migrations/1753106599604-old_something_leftover.js'
 import { UserReceivingInvoiceIdx1753109184611 } from './build/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.js'
+import { AppUserDevice1753285173175 } from './build/src/services/storage/migrations/1753285173175-app_user_device.js'
 
 export default new DataSource({
     type: "better-sqlite3",
@@ -42,10 +44,11 @@ export default new DataSource({
     // logging: true,
     migrations: [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, CreateInviteTokenTable1721751414878,
         PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, DebitToPub1727105758354, UserCbUrl1727112281043,
-        UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611],
+        UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611,
+        AppUserDevice1753285173175],
     entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
         UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder, LndNodeInfo,
-        TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant, AppUserDevice],
+        TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant, AppUserDevice, UserAccess],
     // synchronize: true,
 })
-//npx typeorm migration:generate ./src/services/storage/migrations/app_user_device -d ./datasource.js
+//npx typeorm migration:generate ./src/services/storage/migrations/user_access -d ./datasource.js

src/nostrMiddleware.ts

@@ -12,6 +12,7 @@ export default (serverMethods: Types.ServerMethods, mainHandler: Main, nostrSett
         NostrUserAuthGuard: async (appId, pub) => {
             const app = await mainHandler.storage.applicationStorage.GetApplication(appId || "")
             const nostrUser = await mainHandler.storage.applicationStorage.GetOrCreateNostrAppUser(app, pub || "")
+            await mainHandler.storage.userStorage.UpsertUserAccess(nostrUser.user.user_id, Math.floor(Date.now() / 1000))
             return { user_id: nostrUser.user.user_id, app_user_id: nostrUser.identifier, app_id: appId || "" }
         },
         NostrAdminAuthGuard: async (appId, pub) => {

src/services/main/appUserManager.ts

@@ -5,11 +5,13 @@ import * as Types from '../../../proto/autogenerated/ts/types.js'
 import { MainSettings } from './settings.js'
 import ApplicationManager from './applicationManager.js'
 import { OfferPriceType, ndebitEncode, nmanageEncode, nofferEncode } from '@shocknet/clink-sdk'
+import { getLogger } from '../helpers/logger.js'
 export default class {
 
     storage: Storage
     settings: MainSettings
     applicationManager: ApplicationManager
+    log = getLogger({ component: 'AppUserManager' })
     constructor(storage: Storage, settings: MainSettings, applicationManager: ApplicationManager) {
         this.storage = storage
         this.settings = settings
@@ -30,6 +32,7 @@ export default class {
         if (!decoded.user_id || !decoded.app_id || !decoded.app_user_id) {
             throw new Error("the provided token is not a valid app user token token")
         }
+        this.storage.userStorage.UpsertUserAccess(decoded.user_id, Math.floor(Date.now() / 1000))
         return decoded
     }
 
@@ -117,4 +120,54 @@ export default class {
         const user = await this.storage.applicationStorage.GetApplicationUser(app, ctx.app_user_id);
         await this.storage.applicationStorage.UpdateAppUserMessagingToken(user.identifier, req.device_id, req.firebase_messaging_token);
     }
+
+    async CleanupInactiveUsers() {
+        this.log("Cleaning up inactive users")
+        const inactiveUsers = await this.storage.userStorage.GetInactiveUsers(30)
+        const toDelete:{userId: string, appUserIds: string[]}[] = []
+        for (const u of inactiveUsers) {
+            const user = await this.storage.userStorage.GetUser(u.user_id)
+            if (user.balance_sats > 0) {
+                continue
+            }
+            const txs = await this.storage.paymentStorage.GetUserReceivingTransactions(u.user_id, 0, 1)
+            if (txs.length > 0) {
+                continue
+            }
+            const invoices = await this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(user.serial_id, 0, 0, 1)
+            if (invoices.length > 0) {
+                continue
+            }
+            const userToUser = await this.storage.paymentStorage.GetUserToUserReceivedPayments(u.user_id, 0, 1)
+            if (userToUser.length > 0) {
+                continue
+            }
+            const appUsers = await this.storage.applicationStorage.GetAllAppUsersFromUser(u.user_id)
+            toDelete.push({userId: u.user_id, appUserIds: appUsers.map(a => a.identifier)})
+        }
+        
+        this.log("Found",toDelete.length, "inactive users to delete")
+        // await this.RemoveIntactiveUsers(toDelete) TODO: activate deletion
+    }
+
+    async RemoveIntactiveUsers(toDelete: { userId: string, appUserIds: string[] }[]) {
+        this.log("Deleting",toDelete.length, "inactive users")
+        for (let i = 0; i < toDelete.length; i++) {
+            const {userId,appUserIds} = toDelete[i]
+            this.log("Deleting user", userId, "progress", i+1, "/", toDelete.length)
+            await this.storage.StartTransaction(async tx => {
+                for (const appUserId of appUserIds) {
+                    await this.storage.managementStorage.removeUserGrants(appUserId, tx)
+                    await this.storage.offerStorage.DeleteUserOffers(appUserId, tx)
+                    await this.storage.debitStorage.RemoveUserDebitAccess(appUserId, tx)
+                    await this.storage.applicationStorage.RemoveAppUserDevices(appUserId, tx)
+
+                }
+                await this.storage.paymentStorage.RemoveUserInvoices(userId, tx)
+                await this.storage.productStorage.RemoveUserProducts(userId, tx)
+                await this.storage.paymentStorage.RemoveUserEphemeralKeys(userId, tx)
+            })
+        }
+        this.log("Cleaned up inactive users")
+    }
 }

src/services/main/init.ts

@@ -75,6 +75,7 @@ export const initMainHandler = async (log: PubLogger, mainSettings: MainSettings
         return
     }
     await mainHandler.paymentManager.checkPendingPayments()
+    await mainHandler.appUserManager.CleanupInactiveUsers()
     await mainHandler.paymentManager.watchDog.Start()
     return { mainHandler, apps, liquidityProviderInfo, liquidityProviderApp, wizard, adminManager }
 }

src/services/storage/applicationStorage.ts

@@ -1,5 +1,5 @@
 import crypto from 'crypto';
-import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThanOrEqual } from "typeorm"
+import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThanOrEqual, In } from "typeorm"
 import { generateSecretKey, getPublicKey } from 'nostr-tools';
 import { Application } from "./entity/Application.js"
 import UserStorage from './userStorage.js';
@@ -160,6 +160,12 @@ export default class {
         this.dbs.Remove<User>('User', baseUser, txId)
     }
 
+    async RemoveAppUsersAndBaseUsers(appUserIds: string[],baseUser:string, txId?: string) {
+        await this.dbs.Delete<ApplicationUser>('ApplicationUser', { identifier: In(appUserIds) }, txId)
+        await this.dbs.Delete<User>('User', { user_id: baseUser }, txId)
+
+    }
+
 
     async AddInviteToken(app: Application, sats?: number) {
         return this.dbs.CreateAndSave<InviteToken>('InviteToken', {
@@ -198,4 +204,8 @@ export default class {
     async GetAppUserDevices(appUserId: string, txId?: string) {
         return this.dbs.Find<AppUserDevice>('AppUserDevice', { where: { app_user_id: appUserId } }, txId)
     }
+
+    async RemoveAppUserDevices(appUserId: string, txId?: string) {
+        return this.dbs.Delete<AppUserDevice>('AppUserDevice', { app_user_id: appUserId }, txId)
+    }
 }

src/services/storage/db/db.ts

@@ -28,6 +28,7 @@ import { ManagementGrant } from "../entity/ManagementGrant.js"
 import { ChannelEvent } from "../entity/ChannelEvent.js"
 import { AppUserDevice } from "../entity/AppUserDevice.js"
 import * as fs from 'fs'
+import { UserAccess } from "../entity/UserAccess.js"
 
 
 export type DbSettings = {
@@ -71,7 +72,8 @@ export const MainDbEntities = {
     'UserOffer': UserOffer,
     'Product': Product,
     'ManagementGrant': ManagementGrant,
-    'AppUserDevice': AppUserDevice
+    'AppUserDevice': AppUserDevice,
+    'UserAccess': UserAccess
 }
 export type MainDbNames = keyof typeof MainDbEntities
 export const MainDbEntitiesNames = Object.keys(MainDbEntities)

src/services/storage/debitStorage.ts

@@ -50,4 +50,8 @@ export default class {
     async RemoveDebitAccess(appUserId: string, authorizedPub: string, txId?: string) {
         return this.dbs.Delete<DebitAccess>('DebitAccess', { app_user_id: appUserId, npub: authorizedPub }, txId)
     }
+
+    async RemoveUserDebitAccess(appUserId: string, txId?: string) {
+        return this.dbs.Delete<DebitAccess>('DebitAccess', { app_user_id: appUserId }, txId)
+    }
 }

src/services/storage/entity/UserAccess.ts

@@ -0,0 +1,14 @@
+import { Entity, PrimaryGeneratedColumn, Column, ManyToOne, CreateDateColumn, UpdateDateColumn, PrimaryColumn } from "typeorm"
+import { User } from "./User.js"
+
+@Entity()
+export class UserAccess {
+    @PrimaryColumn()
+    user_id: string
+
+    @Column({ default: 0 })
+    last_seen_at_unix: number
+
+    @Column({ default: false })
+    locked: boolean
+}

src/services/storage/managementStorage.ts

@@ -22,4 +22,8 @@ export class ManagementStorage {
     async removeGrant(appUserId: string, appPubkey: string) {
         return this.dbs.Delete<ManagementGrant>('ManagementGrant', { app_pubkey: appPubkey, app_user_id: appUserId });
     }
-} 
+
+    async removeUserGrants(appUserId: string, txId?: string) {
+        return this.dbs.Delete<ManagementGrant>('ManagementGrant', { app_user_id: appUserId }, txId);
+    }
+}

src/services/storage/migrations/1759426050669-user_access.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class UserAccess1759426050669 implements MigrationInterface {
+    name = 'UserAccess1759426050669'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE TABLE "user_access" ("user_id" varchar PRIMARY KEY NOT NULL, "last_seen_at_unix" integer NOT NULL DEFAULT (0), "locked" boolean NOT NULL DEFAULT (0))`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP TABLE "user_access"`);
+    }
+
+}

src/services/storage/migrations/runner.ts

@@ -23,11 +23,13 @@ import { InvoiceCallbackUrls1752425992291 } from './1752425992291-invoice_callba
 import { AppUserDevice1753285173175 } from './1753285173175-app_user_device.js'
 import { OldSomethingLeftover1753106599604 } from './1753106599604-old_something_leftover.js'
 import { UserReceivingInvoiceIdx1753109184611 } from './1753109184611-user_receiving_invoice_idx.js'
+import { UserAccess1759426050669 } from './1759426050669-user_access.js'
 
 
 export const allMigrations = [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189,
     TrackedProvider1720814323679, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264,
-    DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611, AppUserDevice1753285173175]
+    DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, 
+    InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611, AppUserDevice1753285173175, UserAccess1759426050669]
 
 export const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538, HtlcCount1724266887195, BalanceEvents1724860966825, RootOps1732566440447, RootOpsTime1745428134124, ChannelEvents1750777346411]
 /* export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {

src/services/storage/offerStorage.ts

@@ -27,6 +27,11 @@ export default class {
     async DeleteUserOffer(appUserId: string, offerId: string, txId?: string) {
         await this.dbs.Delete<UserOffer>('UserOffer', { app_user_id: appUserId, offer_id: offerId }, txId)
     }
+
+    async DeleteUserOffers(appUserId: string, txId?: string) {
+        await this.dbs.Delete<UserOffer>('UserOffer', { app_user_id: appUserId }, txId)
+    }
+
     async UpdateUserOffer(app_user_id: string, offerId: string, req: Partial<UserOffer>, txId?: string) {
         return this.dbs.Update<UserOffer>('UserOffer', { app_user_id, offer_id: offerId }, req, txId)
     }

src/services/storage/paymentStorage.ts

@@ -129,6 +129,10 @@ export default class {
         }, txId)
     }
 
+    async RemoveUserInvoices(userId: string, txId?: string) {
+        return this.dbs.Delete<UserReceivingInvoice>('UserReceivingInvoice',  { user: { user_id: userId } }, txId)
+    }
+
     async GetAddressOwner(address: string, txId?: string): Promise<UserReceivingAddress | null> {
         return this.dbs.FindOne<UserReceivingAddress>('UserReceivingAddress', { where: { address } }, txId)
     }
@@ -303,6 +307,10 @@ export default class {
         return found
     }
 
+    async RemoveUserEphemeralKeys(userId: string, txId?: string) {
+        return this.dbs.Delete<UserEphemeralKey>('UserEphemeralKey', { user: { user_id: userId } }, txId)
+    }
+
     async AddPendingUserToUserPayment(fromUserId: string, toUserId: string, amount: number, fee: number, linkedApplication: Application, txId: string) {
         return this.dbs.CreateAndSave<UserToUserPayment>('UserToUserPayment', {
             from_user: await this.userStorage.GetUser(fromUserId, txId),

src/services/storage/productStorage.ts

@@ -19,4 +19,8 @@ export default class {
         }
         return product
     }
+
+    async RemoveUserProducts(userId: string, txId?: string) {
+        return this.dbs.Delete<Product>('Product', { owner: { user_id: userId } }, txId)
+    }
 }

src/services/storage/userStorage.ts

@@ -4,6 +4,8 @@ import { UserBasicAuth } from './entity/UserBasicAuth.js';
 import { getLogger } from '../helpers/logger.js';
 import EventsLogManager from './eventsLog.js';
 import { StorageInterface } from './db/storageInterface.js';
+import { UserAccess } from './entity/UserAccess.js';
+import { LessThan, MoreThan } from 'typeorm';
 export default class {
     dbs: StorageInterface
     eventsLog: EventsLogManager
@@ -113,4 +115,15 @@ export default class {
         const user = await this.GetUser(userId, txId)
         await this.dbs.Update<User>('User', user.serial_id, update, txId)
     }
+
+    async UpsertUserAccess(userId: string, lastSeenAtUnix: number, txId?: string) {
+        return this.dbs.CreateAndSave<UserAccess>('UserAccess', { user_id: userId, last_seen_at_unix: lastSeenAtUnix }, txId)
+    }
+
+    async GetInactiveUsers(inactiveForDays: number) {
+        const seconds = inactiveForDays * 24 * 60 * 60
+        const now = Math.floor(Date.now() / 1000)
+        const lastSeenAtUnix = now - seconds
+        return this.dbs.Find<UserAccess>('UserAccess', { where: { last_seen_at_unix: LessThan(lastSeenAtUnix) } })
+    }
 }