Merge pull request #827 from snake-eaterr/cursor/paid-operations-index

Cursor/paid operations index

datasource.js

@@ -32,13 +32,16 @@ import { UserCbUrl1727112281043 } from './build/src/services/storage/migrations/
 import { UserOffer1733502626042 } from './build/src/services/storage/migrations/1733502626042-user_offer.js'
 import { ManagementGrant1751307732346 } from './build/src/services/storage/migrations/1751307732346-management_grant.js'
 import { InvoiceCallbackUrls1752425992291 } from './build/src/services/storage/migrations/1752425992291-invoice_callback_urls.js'
+import { OldSomethingLeftover1753106599604 } from './build/src/services/storage/migrations/1753106599604-old_something_leftover.js'
+import { UserReceivingInvoiceIdx1753109184611 } from './build/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.js'
+
 export default new DataSource({
     type: "sqlite",
     database: "db.sqlite",
     // logging: true,
     migrations: [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, CreateInviteTokenTable1721751414878,
         PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, DebitToPub1727105758354, UserCbUrl1727112281043,
-        UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291],
+        UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611],
     entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment,
         UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder, LndNodeInfo, TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant],
     // synchronize: true,

proto/autogenerated/client.md

@@ -1224,12 +1224,12 @@ The nostr server will send back a message response, and inside the body there wi
   - __offer_id__: _string_
 
 ### GetUserOperationsRequest
-  - __latestIncomingInvoice__: _number_
-  - __latestIncomingTx__: _number_
-  - __latestIncomingUserToUserPayment__: _number_
-  - __latestOutgoingInvoice__: _number_
-  - __latestOutgoingTx__: _number_
-  - __latestOutgoingUserToUserPayment__: _number_
+  - __latestIncomingInvoice__: _[OperationsCursor](#OperationsCursor)_
+  - __latestIncomingTx__: _[OperationsCursor](#OperationsCursor)_
+  - __latestIncomingUserToUserPayment__: _[OperationsCursor](#OperationsCursor)_
+  - __latestOutgoingInvoice__: _[OperationsCursor](#OperationsCursor)_
+  - __latestOutgoingTx__: _[OperationsCursor](#OperationsCursor)_
+  - __latestOutgoingUserToUserPayment__: _[OperationsCursor](#OperationsCursor)_
   - __max_size__: _number_
 
 ### GetUserOperationsResponse
@@ -1431,6 +1431,10 @@ The nostr server will send back a message response, and inside the body there wi
 ### OpenChannelResponse
   - __channel_id__: _string_
 
+### OperationsCursor
+  - __id__: _number_
+  - __ts__: _number_
+
 ### PayAddressRequest
   - __address__: _string_
   - __amoutSats__: _number_
@@ -1601,9 +1605,9 @@ The nostr server will send back a message response, and inside the body there wi
   - __type__: _[UserOperationType](#UserOperationType)_
 
 ### UserOperations
-  - __fromIndex__: _number_
+  - __fromIndex__: _[OperationsCursor](#OperationsCursor)_
   - __operations__: ARRAY of: _[UserOperation](#UserOperation)_
-  - __toIndex__: _number_
+  - __toIndex__: _[OperationsCursor](#OperationsCursor)_
 
 ### UsersInfo
   - __always_been_inactive__: _number_

proto/autogenerated/go/types.go

@@ -309,13 +309,13 @@ type GetUserOfferInvoicesReq struct {
 	Offer_id       string `json:"offer_id"`
 }
 type GetUserOperationsRequest struct {
-	Latestincominginvoice           int64 `json:"latestIncomingInvoice"`
-	Latestincomingtx                int64 `json:"latestIncomingTx"`
-	Latestincomingusertouserpayment int64 `json:"latestIncomingUserToUserPayment"`
-	Latestoutgoinginvoice           int64 `json:"latestOutgoingInvoice"`
-	Latestoutgoingtx                int64 `json:"latestOutgoingTx"`
-	Latestoutgoingusertouserpayment int64 `json:"latestOutgoingUserToUserPayment"`
-	Max_size                        int64 `json:"max_size"`
+	Latestincominginvoice           *OperationsCursor `json:"latestIncomingInvoice"`
+	Latestincomingtx                *OperationsCursor `json:"latestIncomingTx"`
+	Latestincomingusertouserpayment *OperationsCursor `json:"latestIncomingUserToUserPayment"`
+	Latestoutgoinginvoice           *OperationsCursor `json:"latestOutgoingInvoice"`
+	Latestoutgoingtx                *OperationsCursor `json:"latestOutgoingTx"`
+	Latestoutgoingusertouserpayment *OperationsCursor `json:"latestOutgoingUserToUserPayment"`
+	Max_size                        int64             `json:"max_size"`
 }
 type GetUserOperationsResponse struct {
 	Latestincominginvoiceoperations  *UserOperations `json:"latestIncomingInvoiceOperations"`
@@ -516,6 +516,10 @@ type OpenChannelRequest struct {
 type OpenChannelResponse struct {
 	Channel_id string `json:"channel_id"`
 }
+type OperationsCursor struct {
+	Id int64 `json:"id"`
+	Ts int64 `json:"ts"`
+}
 type PayAddressRequest struct {
 	Address      string `json:"address"`
 	Amoutsats    int64  `json:"amoutSats"`
@@ -686,9 +690,9 @@ type UserOperation struct {
 	Type        UserOperationType `json:"type"`
 }
 type UserOperations struct {
-	Fromindex  int64           `json:"fromIndex"`
-	Operations []UserOperation `json:"operations"`
-	Toindex    int64           `json:"toIndex"`
+	Fromindex  *OperationsCursor `json:"fromIndex"`
+	Operations []UserOperation   `json:"operations"`
+	Toindex    *OperationsCursor `json:"toIndex"`
 }
 type UsersInfo struct {
 	Always_been_inactive int64 `json:"always_been_inactive"`

proto/autogenerated/ts/types.ts

@@ -1775,46 +1775,52 @@ export const GetUserOfferInvoicesReqValidate = (o?: GetUserOfferInvoicesReq, opt
 }
 
 export type GetUserOperationsRequest = {
-    latestIncomingInvoice: number
-    latestIncomingTx: number
-    latestIncomingUserToUserPayment: number
-    latestOutgoingInvoice: number
-    latestOutgoingTx: number
-    latestOutgoingUserToUserPayment: number
+    latestIncomingInvoice: OperationsCursor
+    latestIncomingTx: OperationsCursor
+    latestIncomingUserToUserPayment: OperationsCursor
+    latestOutgoingInvoice: OperationsCursor
+    latestOutgoingTx: OperationsCursor
+    latestOutgoingUserToUserPayment: OperationsCursor
     max_size: number
 }
 export const GetUserOperationsRequestOptionalFields: [] = []
 export type GetUserOperationsRequestOptions = OptionsBaseMessage & {
     checkOptionalsAreSet?: []
-    latestIncomingInvoice_CustomCheck?: (v: number) => boolean
-    latestIncomingTx_CustomCheck?: (v: number) => boolean
-    latestIncomingUserToUserPayment_CustomCheck?: (v: number) => boolean
-    latestOutgoingInvoice_CustomCheck?: (v: number) => boolean
-    latestOutgoingTx_CustomCheck?: (v: number) => boolean
-    latestOutgoingUserToUserPayment_CustomCheck?: (v: number) => boolean
+    latestIncomingInvoice_Options?: OperationsCursorOptions
+    latestIncomingTx_Options?: OperationsCursorOptions
+    latestIncomingUserToUserPayment_Options?: OperationsCursorOptions
+    latestOutgoingInvoice_Options?: OperationsCursorOptions
+    latestOutgoingTx_Options?: OperationsCursorOptions
+    latestOutgoingUserToUserPayment_Options?: OperationsCursorOptions
     max_size_CustomCheck?: (v: number) => boolean
 }
 export const GetUserOperationsRequestValidate = (o?: GetUserOperationsRequest, opts: GetUserOperationsRequestOptions = {}, path: string = 'GetUserOperationsRequest::root.'): Error | null => {
     if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
     if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
 
-    if (typeof o.latestIncomingInvoice !== 'number') return new Error(`${path}.latestIncomingInvoice: is not a number`)
-    if (opts.latestIncomingInvoice_CustomCheck && !opts.latestIncomingInvoice_CustomCheck(o.latestIncomingInvoice)) return new Error(`${path}.latestIncomingInvoice: custom check failed`)
+    const latestIncomingInvoiceErr = OperationsCursorValidate(o.latestIncomingInvoice, opts.latestIncomingInvoice_Options, `${path}.latestIncomingInvoice`)
+    if (latestIncomingInvoiceErr !== null) return latestIncomingInvoiceErr
     
-    if (typeof o.latestIncomingTx !== 'number') return new Error(`${path}.latestIncomingTx: is not a number`)
-    if (opts.latestIncomingTx_CustomCheck && !opts.latestIncomingTx_CustomCheck(o.latestIncomingTx)) return new Error(`${path}.latestIncomingTx: custom check failed`)
 
-    if (typeof o.latestIncomingUserToUserPayment !== 'number') return new Error(`${path}.latestIncomingUserToUserPayment: is not a number`)
-    if (opts.latestIncomingUserToUserPayment_CustomCheck && !opts.latestIncomingUserToUserPayment_CustomCheck(o.latestIncomingUserToUserPayment)) return new Error(`${path}.latestIncomingUserToUserPayment: custom check failed`)
+    const latestIncomingTxErr = OperationsCursorValidate(o.latestIncomingTx, opts.latestIncomingTx_Options, `${path}.latestIncomingTx`)
+    if (latestIncomingTxErr !== null) return latestIncomingTxErr
     
-    if (typeof o.latestOutgoingInvoice !== 'number') return new Error(`${path}.latestOutgoingInvoice: is not a number`)
-    if (opts.latestOutgoingInvoice_CustomCheck && !opts.latestOutgoingInvoice_CustomCheck(o.latestOutgoingInvoice)) return new Error(`${path}.latestOutgoingInvoice: custom check failed`)
 
-    if (typeof o.latestOutgoingTx !== 'number') return new Error(`${path}.latestOutgoingTx: is not a number`)
-    if (opts.latestOutgoingTx_CustomCheck && !opts.latestOutgoingTx_CustomCheck(o.latestOutgoingTx)) return new Error(`${path}.latestOutgoingTx: custom check failed`)
+    const latestIncomingUserToUserPaymentErr = OperationsCursorValidate(o.latestIncomingUserToUserPayment, opts.latestIncomingUserToUserPayment_Options, `${path}.latestIncomingUserToUserPayment`)
+    if (latestIncomingUserToUserPaymentErr !== null) return latestIncomingUserToUserPaymentErr
+    
+
+    const latestOutgoingInvoiceErr = OperationsCursorValidate(o.latestOutgoingInvoice, opts.latestOutgoingInvoice_Options, `${path}.latestOutgoingInvoice`)
+    if (latestOutgoingInvoiceErr !== null) return latestOutgoingInvoiceErr
+    
+
+    const latestOutgoingTxErr = OperationsCursorValidate(o.latestOutgoingTx, opts.latestOutgoingTx_Options, `${path}.latestOutgoingTx`)
+    if (latestOutgoingTxErr !== null) return latestOutgoingTxErr
+    
+
+    const latestOutgoingUserToUserPaymentErr = OperationsCursorValidate(o.latestOutgoingUserToUserPayment, opts.latestOutgoingUserToUserPayment_Options, `${path}.latestOutgoingUserToUserPayment`)
+    if (latestOutgoingUserToUserPaymentErr !== null) return latestOutgoingUserToUserPaymentErr
     
-    if (typeof o.latestOutgoingUserToUserPayment !== 'number') return new Error(`${path}.latestOutgoingUserToUserPayment: is not a number`)
-    if (opts.latestOutgoingUserToUserPayment_CustomCheck && !opts.latestOutgoingUserToUserPayment_CustomCheck(o.latestOutgoingUserToUserPayment)) return new Error(`${path}.latestOutgoingUserToUserPayment: custom check failed`)
 
     if (typeof o.max_size !== 'number') return new Error(`${path}.max_size: is not a number`)
     if (opts.max_size_CustomCheck && !opts.max_size_CustomCheck(o.max_size)) return new Error(`${path}.max_size: custom check failed`)
@@ -3031,6 +3037,29 @@ export const OpenChannelResponseValidate = (o?: OpenChannelResponse, opts: OpenC
     return null
 }
 
+export type OperationsCursor = {
+    id: number
+    ts: number
+}
+export const OperationsCursorOptionalFields: [] = []
+export type OperationsCursorOptions = OptionsBaseMessage & {
+    checkOptionalsAreSet?: []
+    id_CustomCheck?: (v: number) => boolean
+    ts_CustomCheck?: (v: number) => boolean
+}
+export const OperationsCursorValidate = (o?: OperationsCursor, opts: OperationsCursorOptions = {}, path: string = 'OperationsCursor::root.'): Error | null => {
+    if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
+    if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
+
+    if (typeof o.id !== 'number') return new Error(`${path}.id: is not a number`)
+    if (opts.id_CustomCheck && !opts.id_CustomCheck(o.id)) return new Error(`${path}.id: custom check failed`)
+
+    if (typeof o.ts !== 'number') return new Error(`${path}.ts: is not a number`)
+    if (opts.ts_CustomCheck && !opts.ts_CustomCheck(o.ts)) return new Error(`${path}.ts: custom check failed`)
+
+    return null
+}
+
 export type PayAddressRequest = {
     address: string
     amoutSats: number
@@ -3998,24 +4027,25 @@ export const UserOperationValidate = (o?: UserOperation, opts: UserOperationOpti
 }
 
 export type UserOperations = {
-    fromIndex: number
+    fromIndex: OperationsCursor
     operations: UserOperation[]
-    toIndex: number
+    toIndex: OperationsCursor
 }
 export const UserOperationsOptionalFields: [] = []
 export type UserOperationsOptions = OptionsBaseMessage & {
     checkOptionalsAreSet?: []
-    fromIndex_CustomCheck?: (v: number) => boolean
+    fromIndex_Options?: OperationsCursorOptions
     operations_ItemOptions?: UserOperationOptions
     operations_CustomCheck?: (v: UserOperation[]) => boolean
-    toIndex_CustomCheck?: (v: number) => boolean
+    toIndex_Options?: OperationsCursorOptions
 }
 export const UserOperationsValidate = (o?: UserOperations, opts: UserOperationsOptions = {}, path: string = 'UserOperations::root.'): Error | null => {
     if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message')
     if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null')
 
-    if (typeof o.fromIndex !== 'number') return new Error(`${path}.fromIndex: is not a number`)
-    if (opts.fromIndex_CustomCheck && !opts.fromIndex_CustomCheck(o.fromIndex)) return new Error(`${path}.fromIndex: custom check failed`)
+    const fromIndexErr = OperationsCursorValidate(o.fromIndex, opts.fromIndex_Options, `${path}.fromIndex`)
+    if (fromIndexErr !== null) return fromIndexErr
+    
 
     if (!Array.isArray(o.operations)) return new Error(`${path}.operations: is not an array`)
     for (let index = 0; index < o.operations.length; index++) {
@@ -4024,8 +4054,9 @@ export const UserOperationsValidate = (o?: UserOperations, opts: UserOperationsO
     }
     if (opts.operations_CustomCheck && !opts.operations_CustomCheck(o.operations)) return new Error(`${path}.operations: custom check failed`)
 
-    if (typeof o.toIndex !== 'number') return new Error(`${path}.toIndex: is not a number`)
-    if (opts.toIndex_CustomCheck && !opts.toIndex_CustomCheck(o.toIndex)) return new Error(`${path}.toIndex: custom check failed`)
+    const toIndexErr = OperationsCursorValidate(o.toIndex, opts.toIndex_Options, `${path}.toIndex`)
+    if (toIndexErr !== null) return toIndexErr
+    
 
     return null
 }

proto/service/structs.proto

@@ -534,13 +534,19 @@ message UserInfo{
     string bridge_url = 11;
     string nmanage = 12;
 }
+
+
+message OperationsCursor {
+    int64 ts = 1;       // last timestamp
+    int64 id = 2;       // last serial_id
+}
 message GetUserOperationsRequest{
-    int64 latestIncomingInvoice = 1;
-    int64 latestOutgoingInvoice = 2;
-    int64 latestIncomingTx = 3;
-    int64 latestOutgoingTx = 4;
-    int64 latestIncomingUserToUserPayment = 5;
-    int64 latestOutgoingUserToUserPayment = 6;
+    OperationsCursor latestIncomingInvoice = 1;
+    OperationsCursor latestOutgoingInvoice = 2;
+    OperationsCursor latestIncomingTx = 3;
+    OperationsCursor latestOutgoingTx = 4;
+    OperationsCursor latestIncomingUserToUserPayment = 5;
+    OperationsCursor latestOutgoingUserToUserPayment = 6;
     int64 max_size = 7;
 }
 enum UserOperationType {
@@ -566,8 +572,8 @@ message UserOperation {
     bool internal = 11;
 }
 message UserOperations {
-    int64 fromIndex=1;
-    int64 toIndex=2;
+    OperationsCursor fromIndex=1;
+    OperationsCursor toIndex=2;
     repeated UserOperation operations=3;
 }
 message GetUserOperationsResponse{

src/services/main/liquidityProvider.ts

@@ -227,9 +227,9 @@ export class LiquidityProvider {
             throw new Error("liquidity provider is not ready yet")
         }
         const res = await this.client.GetUserOperations({
-            latestIncomingInvoice: 0, latestOutgoingInvoice: 0,
-            latestIncomingTx: 0, latestOutgoingTx: 0, latestIncomingUserToUserPayment: 0,
-            latestOutgoingUserToUserPayment: 0, max_size: 200
+            latestIncomingInvoice: { ts: 0, id: 0 }, latestOutgoingInvoice: { ts: 0, id: 0 },
+            latestIncomingTx: { ts: 0, id: 0 }, latestOutgoingTx: { ts: 0, id: 0 }, latestIncomingUserToUserPayment: { ts: 0, id: 0 },
+            latestOutgoingUserToUserPayment: { ts: 0, id: 0 }, max_size: 200
         })
         if (res.status === 'ERROR') {
             this.log("error getting operations", res.reason)

src/services/main/paymentManager.ts

@@ -626,14 +626,15 @@ export default class {
     mapOperations(operations: UserOperationInfo[], type: Types.UserOperationType, inbound: boolean): Types.UserOperations {
         if (operations.length === 0) {
             return {
-                fromIndex: 0,
-                toIndex: 0,
+                fromIndex: { ts: 0, id: 0 },
+                toIndex: { ts: 0, id: 0 },
                 operations: []
             }
         }
         return {
-            toIndex: operations[0].serial_id,
-            fromIndex: operations[operations.length - 1].serial_id,
+            // We fetch in ascending order
+            toIndex: { ts: operations.at(-1)!.paid_at_unix, id:  operations.at(-1)!.serial_id } ,
+            fromIndex: { ts: operations[0].paid_at_unix, id:  operations[0]!.serial_id },
             operations: operations.map((o: UserOperationInfo): Types.UserOperation => {
                 let identifier = "";
                 if (o.invoice) {
@@ -687,12 +688,12 @@ export default class {
             throw new Error("user is banned, cannot retrieve operations")
         }
         const [outgoingInvoices, outgoingTransactions, incomingInvoices, incomingTransactions, incomingUserToUser, outgoingUserToUser] = await Promise.all([
-            this.storage.paymentStorage.GetUserInvoicePayments(userId, req.latestOutgoingInvoice, req.max_size), //
-            this.storage.paymentStorage.GetUserTransactionPayments(userId, req.latestOutgoingTx, req.max_size),
-            this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(userId, req.latestIncomingInvoice, req.max_size),
-            this.storage.paymentStorage.GetUserReceivingTransactions(userId, req.latestIncomingTx, req.max_size),
-            this.storage.paymentStorage.GetUserToUserReceivedPayments(userId, req.latestIncomingUserToUserPayment, req.max_size),
-            this.storage.paymentStorage.GetUserToUserSentPayments(userId, req.latestOutgoingUserToUserPayment, req.max_size)
+            this.storage.paymentStorage.GetUserInvoicePayments(userId, req.latestOutgoingInvoice.id, req.max_size), //
+            this.storage.paymentStorage.GetUserTransactionPayments(userId, req.latestOutgoingTx.id, req.max_size),
+            this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(user.serial_id, req.latestIncomingInvoice.id, req.latestIncomingInvoice.ts, req.max_size),
+            this.storage.paymentStorage.GetUserReceivingTransactions(userId, req.latestIncomingTx.id, req.max_size),
+            this.storage.paymentStorage.GetUserToUserReceivedPayments(userId, req.latestIncomingUserToUserPayment.id, req.max_size),
+            this.storage.paymentStorage.GetUserToUserSentPayments(userId, req.latestOutgoingUserToUserPayment.id, req.max_size)
         ])
         return {
             latestIncomingInvoiceOperations: this.mapOperations(incomingInvoices, Types.UserOperationType.INCOMING_INVOICE, true),

src/services/storage/db/serializationHelpers.ts

@@ -1,4 +1,4 @@
-import { FindOperator, LessThan, MoreThan, LessThanOrEqual, MoreThanOrEqual, Equal, Like, ILike, Between, In, Any, IsNull, Not, FindOptionsWhere } from 'typeorm';
+import { FindOperator, LessThan, MoreThan, LessThanOrEqual, MoreThanOrEqual, Equal, Like, ILike, Between, In, Any, IsNull, Not, FindOptionsWhere, And } from 'typeorm';
 export type WhereCondition<T> = FindOptionsWhere<T> | FindOptionsWhere<T>[]
 
 type SerializedFindOperator = {
@@ -11,7 +11,7 @@ export function serializeFindOperator(operator: FindOperator<any>): SerializedFi
     return {
         _type: 'FindOperator',
         type: operator['type'],
-        value: operator['value'],
+        value: Array.isArray(operator['value']) ? operator["value"].map(serializeFindOperator) : operator["value"],
     };
 }
 
@@ -41,6 +41,11 @@ export function deserializeFindOperator(serialized: SerializedFindOperator): Fin
             return IsNull();
         case 'not':
             return Not(deserializeFindOperator(serialized.value));
+        case 'and':
+            if (serialized.value.length !== 2) {
+                throw new Error("Typeorm and operator with more than 2 arguments not supported");
+            }
+            return And(deserializeFindOperator(serialized.value[0]), deserializeFindOperator(serialized.value[1]));
         default:
             throw new Error(`Unknown FindOperator type: ${serialized.type}`);
     }

src/services/storage/entity/UserReceivingInvoice.ts

@@ -9,6 +9,7 @@ export type ZapInfo = {
     description: string
 }
 @Entity()
+@Index("recv_invoice_paid_serial", ["user.serial_id", "paid_at_unix", "serial_id"], { where: "paid_at_unix > 0" })
 export class UserReceivingInvoice {
 
     @PrimaryGeneratedColumn()

src/services/storage/migrations/1753106599604-old_something_leftover.ts

@@ -0,0 +1,27 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+
+// Remove the old left over column "something" from user_transaction_payment
+export class OldSomethingLeftover1753106599604 implements MigrationInterface {
+    name = 'OldSomethingLeftover1753106599604'
+    
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "user_transaction_unique"`);
+        await queryRunner.query(`CREATE TABLE "temporary_user_transaction_payment" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "address" varchar NOT NULL, "tx_hash" varchar NOT NULL, "output_index" integer NOT NULL, "paid_amount" integer NOT NULL, "chain_fees" integer NOT NULL, "service_fees" integer NOT NULL, "paid_at_unix" integer NOT NULL, "internal" boolean NOT NULL DEFAULT (0), "confs" integer NOT NULL DEFAULT (0), "broadcast_height" integer NOT NULL DEFAULT (0), "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')), "userSerialId" integer, "linkedApplicationSerialId" integer, CONSTRAINT "FK_75abdd29270979e901da0dba7b9" FOREIGN KEY ("linkedApplicationSerialId") REFERENCES "application" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_6f24c901b4103f7146eb615a5db" FOREIGN KEY ("userSerialId") REFERENCES "user" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "temporary_user_transaction_payment"("serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId") SELECT "serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId" FROM "user_transaction_payment"`);
+        await queryRunner.query(`DROP TABLE "user_transaction_payment"`);
+        await queryRunner.query(`ALTER TABLE "temporary_user_transaction_payment" RENAME TO "user_transaction_payment"`);
+        await queryRunner.query(`CREATE UNIQUE INDEX "user_transaction_unique" ON "user_transaction_payment" ("tx_hash", "output_index") `);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "user_transaction_unique"`);
+        await queryRunner.query(`ALTER TABLE "user_transaction_payment" RENAME TO "temporary_user_transaction_payment"`);
+        await queryRunner.query(`CREATE TABLE "user_transaction_payment" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "address" varchar NOT NULL, "tx_hash" varchar NOT NULL, "output_index" integer NOT NULL, "paid_amount" integer NOT NULL, "chain_fees" integer NOT NULL, "service_fees" integer NOT NULL, "paid_at_unix" integer NOT NULL, "internal" boolean NOT NULL DEFAULT (0), "confs" integer NOT NULL DEFAULT (0), "broadcast_height" integer NOT NULL DEFAULT (0), "something" integer NOT NULL DEFAULT (0), "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')), "userSerialId" integer, "linkedApplicationSerialId" integer, CONSTRAINT "FK_75abdd29270979e901da0dba7b9" FOREIGN KEY ("linkedApplicationSerialId") REFERENCES "application" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_6f24c901b4103f7146eb615a5db" FOREIGN KEY ("userSerialId") REFERENCES "user" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION)`);
+        await queryRunner.query(`INSERT INTO "user_transaction_payment"("serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId") SELECT "serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId" FROM "temporary_user_transaction_payment"`);
+        await queryRunner.query(`DROP TABLE "temporary_user_transaction_payment"`);
+        await queryRunner.query(`CREATE UNIQUE INDEX "user_transaction_unique" ON "user_transaction_payment" ("tx_hash", "output_index") `);
+    }
+
+}

src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.ts

@@ -0,0 +1,14 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class UserReceivingInvoiceIdx1753109184611 implements MigrationInterface {
+    name = 'UserReceivingInvoiceIdx1753109184611'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX "recv_invoice_paid_serial" ON "user_receiving_invoice" ("userSerialId", "paid_at_unix", "serial_id") WHERE paid_at_unix > 0`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "recv_invoice_paid_serial"`);
+    }
+
+}

src/services/storage/migrations/runner.ts

@@ -20,9 +20,11 @@ import { ChannelEvents1750777346411 } from './1750777346411-channel_events.js'
 import { ManagementGrant1751307732346 } from './1751307732346-management_grant.js'
 import { ManagementGrantBanned1751989251513 } from './1751989251513-management_grant_banned.js'
 import { InvoiceCallbackUrls1752425992291 } from './1752425992291-invoice_callback_urls.js'
+import { OldSomethingLeftover1753106599604 } from './1753106599604-old_something_leftover.js'
+import { UserReceivingInvoiceIdx1753109184611 } from './1753109184611-user_receiving_invoice_idx.js'
 export const allMigrations = [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189,
     TrackedProvider1720814323679, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264,
-    DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291]
+    DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611]
 export const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538, HtlcCount1724266887195, BalanceEvents1724860966825, RootOps1732566440447, RootOpsTime1745428134124, ChannelEvents1750777346411]
 /* export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise<boolean> => {
     await connectAndMigrate(log, storageManager, allMigrations, allMetricsMigrations)

src/services/storage/paymentStorage.ts

@@ -1,5 +1,5 @@
 import crypto from 'crypto';
-import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThan, MoreThanOrEqual, Not } from "typeorm"
+import { And, Between, Equal, FindOperator, IsNull, LessThanOrEqual, MoreThan, MoreThanOrEqual } from "typeorm"
 import { User } from './entity/User.js';
 import { UserTransactionPayment } from './entity/UserTransactionPayment.js';
 import { EphemeralKeyType, UserEphemeralKey } from './entity/UserEphemeralKey.js';
@@ -42,11 +42,11 @@ export default class {
         return this.dbs.Find<AddressReceivingTransaction>('AddressReceivingTransaction', {
             where: {
                 user_address: { user: { user_id: userId } },
-                serial_id: MoreThanOrEqual(fromIndex),
+                serial_id: MoreThan(fromIndex),
                 paid_at_unix: MoreThan(0),
             },
             order: {
-                paid_at_unix: 'DESC'
+                paid_at_unix: 'ASC'
             },
             take
         }, txId)
@@ -73,20 +73,42 @@ export default class {
         return this.dbs.Update<UserReceivingInvoice>('UserReceivingInvoice', invoice.serial_id, i, txId)
     }
 
-    GetUserInvoicesFlaggedAsPaid(userId: string, fromIndex: number, take = 50, txId?: string): Promise<UserReceivingInvoice[]> {
-        return this.dbs.Find<UserReceivingInvoice>('UserReceivingInvoice', {
-            where: {
-                user: {
-                    user_id: userId
+    async GetUserInvoicesFlaggedAsPaid(userSerialId: number, fromIndex: number, fromPaidTimestamp: number, take = 50, txId?: string): Promise<UserReceivingInvoice[]> {
+        let items: UserReceivingInvoice[] = [];
+        if (fromPaidTimestamp > 0) {
+            // First fetch same paid_at_unix, higher serial_id
+            const firstBatch = await this.dbs.Find<UserReceivingInvoice>('UserReceivingInvoice', {
+                where: {
+                    user: { serial_id: userSerialId },
+                    paid_at_unix: And(MoreThan(0), Equal(fromPaidTimestamp)),
+                    serial_id: MoreThan(fromIndex)
                 },
-                serial_id: MoreThanOrEqual(fromIndex),
-                paid_at_unix: MoreThan(0),
-            },
-            order: {
-                paid_at_unix: 'DESC'
-            },
-            take
-        }, txId)
+                order: {
+                    paid_at_unix: 'ASC',
+                    serial_id: 'ASC'
+                },
+                take
+            }, txId);
+            items.push(...firstBatch);
+        } 
+
+        const needMore = take - items.length
+        // If need more, fetch higher paid_at_unix
+        if (needMore > 0) {
+            const secondBatch = await this.dbs.Find<UserReceivingInvoice>('UserReceivingInvoice', {
+                where: {
+                    user: { serial_id: userSerialId },
+                    paid_at_unix: And(MoreThan(0), MoreThan(fromPaidTimestamp)),
+                },
+                order: {
+                    paid_at_unix: 'ASC',
+                    serial_id: 'ASC'
+                },
+                take: needMore
+            }, txId)
+            items.push(...secondBatch)
+        }
+        return items
     }
 
     async AddUserInvoice(user: User, invoice: string, options: InboundOptionals = { expiry: defaultInvoiceExpiry }, providerDestination?: string, txId?: string): Promise<UserReceivingInvoice> {
@@ -184,11 +206,11 @@ export default class {
                 user: {
                     user_id: userId
                 },
-                serial_id: MoreThanOrEqual(fromIndex),
+                serial_id: MoreThan(fromIndex),
                 paid_at_unix: MoreThan(-1),
             },
             order: {
-                paid_at_unix: 'DESC'
+                paid_at_unix: 'ASC'
             },
             take
         }, txId)
@@ -232,11 +254,11 @@ export default class {
                 user: {
                     user_id: userId
                 },
-                serial_id: MoreThanOrEqual(fromIndex),
+                serial_id: MoreThan(fromIndex),
                 paid_at_unix: MoreThan(0),
             },
             order: {
-                paid_at_unix: 'DESC'
+                paid_at_unix: 'ASC'
             },
             take
         }, txId)
@@ -301,11 +323,11 @@ export default class {
                 to_user: {
                     user_id: userId
                 },
-                serial_id: MoreThanOrEqual(fromIndex),
+                serial_id: MoreThan(fromIndex),
                 paid_at_unix: MoreThan(0),
             },
             order: {
-                paid_at_unix: 'DESC'
+                paid_at_unix: 'ASC'
             },
             take
         }, txId)
@@ -318,11 +340,11 @@ export default class {
                 from_user: {
                     user_id: userId
                 },
-                serial_id: MoreThanOrEqual(fromIndex),
+                serial_id: MoreThan(fromIndex),
                 paid_at_unix: MoreThan(0),
             },
             order: {
-                paid_at_unix: 'DESC'
+                paid_at_unix: 'ASC'
             },
             take
         }, txId)