diff --git a/datasource.js b/datasource.js index 08db3ca7..3d890917 100644 --- a/datasource.js +++ b/datasource.js @@ -32,13 +32,16 @@ import { UserCbUrl1727112281043 } from './build/src/services/storage/migrations/ import { UserOffer1733502626042 } from './build/src/services/storage/migrations/1733502626042-user_offer.js' import { ManagementGrant1751307732346 } from './build/src/services/storage/migrations/1751307732346-management_grant.js' import { InvoiceCallbackUrls1752425992291 } from './build/src/services/storage/migrations/1752425992291-invoice_callback_urls.js' +import { OldSomethingLeftover1753106599604 } from './build/src/services/storage/migrations/1753106599604-old_something_leftover.js' +import { UserReceivingInvoiceIdx1753109184611 } from './build/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.js' + export default new DataSource({ type: "sqlite", database: "db.sqlite", // logging: true, migrations: [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, DebitToPub1727105758354, UserCbUrl1727112281043, - UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291], + UserOffer1733502626042, ManagementGrant1751307732346, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611], entities: [User, UserReceivingInvoice, UserReceivingAddress, AddressReceivingTransaction, UserInvoicePayment, UserTransactionPayment, UserBasicAuth, UserEphemeralKey, Product, UserToUserPayment, Application, ApplicationUser, UserToUserPayment, LspOrder, LndNodeInfo, TrackedProvider, InviteToken, DebitAccess, UserOffer, ManagementGrant], // synchronize: true, diff --git a/proto/autogenerated/client.md b/proto/autogenerated/client.md index fff56015..e85276c2 100644 --- a/proto/autogenerated/client.md +++ b/proto/autogenerated/client.md @@ -1224,12 +1224,12 @@ The nostr server will send back a message response, and inside the body there wi - __offer_id__: _string_ ### GetUserOperationsRequest - - __latestIncomingInvoice__: _number_ - - __latestIncomingTx__: _number_ - - __latestIncomingUserToUserPayment__: _number_ - - __latestOutgoingInvoice__: _number_ - - __latestOutgoingTx__: _number_ - - __latestOutgoingUserToUserPayment__: _number_ + - __latestIncomingInvoice__: _[OperationsCursor](#OperationsCursor)_ + - __latestIncomingTx__: _[OperationsCursor](#OperationsCursor)_ + - __latestIncomingUserToUserPayment__: _[OperationsCursor](#OperationsCursor)_ + - __latestOutgoingInvoice__: _[OperationsCursor](#OperationsCursor)_ + - __latestOutgoingTx__: _[OperationsCursor](#OperationsCursor)_ + - __latestOutgoingUserToUserPayment__: _[OperationsCursor](#OperationsCursor)_ - __max_size__: _number_ ### GetUserOperationsResponse @@ -1431,6 +1431,10 @@ The nostr server will send back a message response, and inside the body there wi ### OpenChannelResponse - __channel_id__: _string_ +### OperationsCursor + - __id__: _number_ + - __ts__: _number_ + ### PayAddressRequest - __address__: _string_ - __amoutSats__: _number_ @@ -1601,9 +1605,9 @@ The nostr server will send back a message response, and inside the body there wi - __type__: _[UserOperationType](#UserOperationType)_ ### UserOperations - - __fromIndex__: _number_ + - __fromIndex__: _[OperationsCursor](#OperationsCursor)_ - __operations__: ARRAY of: _[UserOperation](#UserOperation)_ - - __toIndex__: _number_ + - __toIndex__: _[OperationsCursor](#OperationsCursor)_ ### UsersInfo - __always_been_inactive__: _number_ diff --git a/proto/autogenerated/go/types.go b/proto/autogenerated/go/types.go index 7e8daa6d..a3cca9f2 100644 --- a/proto/autogenerated/go/types.go +++ b/proto/autogenerated/go/types.go @@ -309,13 +309,13 @@ type GetUserOfferInvoicesReq struct { Offer_id string `json:"offer_id"` } type GetUserOperationsRequest struct { - Latestincominginvoice int64 `json:"latestIncomingInvoice"` - Latestincomingtx int64 `json:"latestIncomingTx"` - Latestincomingusertouserpayment int64 `json:"latestIncomingUserToUserPayment"` - Latestoutgoinginvoice int64 `json:"latestOutgoingInvoice"` - Latestoutgoingtx int64 `json:"latestOutgoingTx"` - Latestoutgoingusertouserpayment int64 `json:"latestOutgoingUserToUserPayment"` - Max_size int64 `json:"max_size"` + Latestincominginvoice *OperationsCursor `json:"latestIncomingInvoice"` + Latestincomingtx *OperationsCursor `json:"latestIncomingTx"` + Latestincomingusertouserpayment *OperationsCursor `json:"latestIncomingUserToUserPayment"` + Latestoutgoinginvoice *OperationsCursor `json:"latestOutgoingInvoice"` + Latestoutgoingtx *OperationsCursor `json:"latestOutgoingTx"` + Latestoutgoingusertouserpayment *OperationsCursor `json:"latestOutgoingUserToUserPayment"` + Max_size int64 `json:"max_size"` } type GetUserOperationsResponse struct { Latestincominginvoiceoperations *UserOperations `json:"latestIncomingInvoiceOperations"` @@ -516,6 +516,10 @@ type OpenChannelRequest struct { type OpenChannelResponse struct { Channel_id string `json:"channel_id"` } +type OperationsCursor struct { + Id int64 `json:"id"` + Ts int64 `json:"ts"` +} type PayAddressRequest struct { Address string `json:"address"` Amoutsats int64 `json:"amoutSats"` @@ -686,9 +690,9 @@ type UserOperation struct { Type UserOperationType `json:"type"` } type UserOperations struct { - Fromindex int64 `json:"fromIndex"` - Operations []UserOperation `json:"operations"` - Toindex int64 `json:"toIndex"` + Fromindex *OperationsCursor `json:"fromIndex"` + Operations []UserOperation `json:"operations"` + Toindex *OperationsCursor `json:"toIndex"` } type UsersInfo struct { Always_been_inactive int64 `json:"always_been_inactive"` diff --git a/proto/autogenerated/ts/types.ts b/proto/autogenerated/ts/types.ts index cb3f357f..46278ae6 100644 --- a/proto/autogenerated/ts/types.ts +++ b/proto/autogenerated/ts/types.ts @@ -1775,46 +1775,52 @@ export const GetUserOfferInvoicesReqValidate = (o?: GetUserOfferInvoicesReq, opt } export type GetUserOperationsRequest = { - latestIncomingInvoice: number - latestIncomingTx: number - latestIncomingUserToUserPayment: number - latestOutgoingInvoice: number - latestOutgoingTx: number - latestOutgoingUserToUserPayment: number + latestIncomingInvoice: OperationsCursor + latestIncomingTx: OperationsCursor + latestIncomingUserToUserPayment: OperationsCursor + latestOutgoingInvoice: OperationsCursor + latestOutgoingTx: OperationsCursor + latestOutgoingUserToUserPayment: OperationsCursor max_size: number } export const GetUserOperationsRequestOptionalFields: [] = [] export type GetUserOperationsRequestOptions = OptionsBaseMessage & { checkOptionalsAreSet?: [] - latestIncomingInvoice_CustomCheck?: (v: number) => boolean - latestIncomingTx_CustomCheck?: (v: number) => boolean - latestIncomingUserToUserPayment_CustomCheck?: (v: number) => boolean - latestOutgoingInvoice_CustomCheck?: (v: number) => boolean - latestOutgoingTx_CustomCheck?: (v: number) => boolean - latestOutgoingUserToUserPayment_CustomCheck?: (v: number) => boolean + latestIncomingInvoice_Options?: OperationsCursorOptions + latestIncomingTx_Options?: OperationsCursorOptions + latestIncomingUserToUserPayment_Options?: OperationsCursorOptions + latestOutgoingInvoice_Options?: OperationsCursorOptions + latestOutgoingTx_Options?: OperationsCursorOptions + latestOutgoingUserToUserPayment_Options?: OperationsCursorOptions max_size_CustomCheck?: (v: number) => boolean } export const GetUserOperationsRequestValidate = (o?: GetUserOperationsRequest, opts: GetUserOperationsRequestOptions = {}, path: string = 'GetUserOperationsRequest::root.'): Error | null => { if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') - if (typeof o.latestIncomingInvoice !== 'number') return new Error(`${path}.latestIncomingInvoice: is not a number`) - if (opts.latestIncomingInvoice_CustomCheck && !opts.latestIncomingInvoice_CustomCheck(o.latestIncomingInvoice)) return new Error(`${path}.latestIncomingInvoice: custom check failed`) + const latestIncomingInvoiceErr = OperationsCursorValidate(o.latestIncomingInvoice, opts.latestIncomingInvoice_Options, `${path}.latestIncomingInvoice`) + if (latestIncomingInvoiceErr !== null) return latestIncomingInvoiceErr + - if (typeof o.latestIncomingTx !== 'number') return new Error(`${path}.latestIncomingTx: is not a number`) - if (opts.latestIncomingTx_CustomCheck && !opts.latestIncomingTx_CustomCheck(o.latestIncomingTx)) return new Error(`${path}.latestIncomingTx: custom check failed`) + const latestIncomingTxErr = OperationsCursorValidate(o.latestIncomingTx, opts.latestIncomingTx_Options, `${path}.latestIncomingTx`) + if (latestIncomingTxErr !== null) return latestIncomingTxErr + - if (typeof o.latestIncomingUserToUserPayment !== 'number') return new Error(`${path}.latestIncomingUserToUserPayment: is not a number`) - if (opts.latestIncomingUserToUserPayment_CustomCheck && !opts.latestIncomingUserToUserPayment_CustomCheck(o.latestIncomingUserToUserPayment)) return new Error(`${path}.latestIncomingUserToUserPayment: custom check failed`) + const latestIncomingUserToUserPaymentErr = OperationsCursorValidate(o.latestIncomingUserToUserPayment, opts.latestIncomingUserToUserPayment_Options, `${path}.latestIncomingUserToUserPayment`) + if (latestIncomingUserToUserPaymentErr !== null) return latestIncomingUserToUserPaymentErr + - if (typeof o.latestOutgoingInvoice !== 'number') return new Error(`${path}.latestOutgoingInvoice: is not a number`) - if (opts.latestOutgoingInvoice_CustomCheck && !opts.latestOutgoingInvoice_CustomCheck(o.latestOutgoingInvoice)) return new Error(`${path}.latestOutgoingInvoice: custom check failed`) + const latestOutgoingInvoiceErr = OperationsCursorValidate(o.latestOutgoingInvoice, opts.latestOutgoingInvoice_Options, `${path}.latestOutgoingInvoice`) + if (latestOutgoingInvoiceErr !== null) return latestOutgoingInvoiceErr + - if (typeof o.latestOutgoingTx !== 'number') return new Error(`${path}.latestOutgoingTx: is not a number`) - if (opts.latestOutgoingTx_CustomCheck && !opts.latestOutgoingTx_CustomCheck(o.latestOutgoingTx)) return new Error(`${path}.latestOutgoingTx: custom check failed`) + const latestOutgoingTxErr = OperationsCursorValidate(o.latestOutgoingTx, opts.latestOutgoingTx_Options, `${path}.latestOutgoingTx`) + if (latestOutgoingTxErr !== null) return latestOutgoingTxErr + - if (typeof o.latestOutgoingUserToUserPayment !== 'number') return new Error(`${path}.latestOutgoingUserToUserPayment: is not a number`) - if (opts.latestOutgoingUserToUserPayment_CustomCheck && !opts.latestOutgoingUserToUserPayment_CustomCheck(o.latestOutgoingUserToUserPayment)) return new Error(`${path}.latestOutgoingUserToUserPayment: custom check failed`) + const latestOutgoingUserToUserPaymentErr = OperationsCursorValidate(o.latestOutgoingUserToUserPayment, opts.latestOutgoingUserToUserPayment_Options, `${path}.latestOutgoingUserToUserPayment`) + if (latestOutgoingUserToUserPaymentErr !== null) return latestOutgoingUserToUserPaymentErr + if (typeof o.max_size !== 'number') return new Error(`${path}.max_size: is not a number`) if (opts.max_size_CustomCheck && !opts.max_size_CustomCheck(o.max_size)) return new Error(`${path}.max_size: custom check failed`) @@ -3031,6 +3037,29 @@ export const OpenChannelResponseValidate = (o?: OpenChannelResponse, opts: OpenC return null } +export type OperationsCursor = { + id: number + ts: number +} +export const OperationsCursorOptionalFields: [] = [] +export type OperationsCursorOptions = OptionsBaseMessage & { + checkOptionalsAreSet?: [] + id_CustomCheck?: (v: number) => boolean + ts_CustomCheck?: (v: number) => boolean +} +export const OperationsCursorValidate = (o?: OperationsCursor, opts: OperationsCursorOptions = {}, path: string = 'OperationsCursor::root.'): Error | null => { + if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') + if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') + + if (typeof o.id !== 'number') return new Error(`${path}.id: is not a number`) + if (opts.id_CustomCheck && !opts.id_CustomCheck(o.id)) return new Error(`${path}.id: custom check failed`) + + if (typeof o.ts !== 'number') return new Error(`${path}.ts: is not a number`) + if (opts.ts_CustomCheck && !opts.ts_CustomCheck(o.ts)) return new Error(`${path}.ts: custom check failed`) + + return null +} + export type PayAddressRequest = { address: string amoutSats: number @@ -3998,24 +4027,25 @@ export const UserOperationValidate = (o?: UserOperation, opts: UserOperationOpti } export type UserOperations = { - fromIndex: number + fromIndex: OperationsCursor operations: UserOperation[] - toIndex: number + toIndex: OperationsCursor } export const UserOperationsOptionalFields: [] = [] export type UserOperationsOptions = OptionsBaseMessage & { checkOptionalsAreSet?: [] - fromIndex_CustomCheck?: (v: number) => boolean + fromIndex_Options?: OperationsCursorOptions operations_ItemOptions?: UserOperationOptions operations_CustomCheck?: (v: UserOperation[]) => boolean - toIndex_CustomCheck?: (v: number) => boolean + toIndex_Options?: OperationsCursorOptions } export const UserOperationsValidate = (o?: UserOperations, opts: UserOperationsOptions = {}, path: string = 'UserOperations::root.'): Error | null => { if (opts.checkOptionalsAreSet && opts.allOptionalsAreSet) return new Error(path + ': only one of checkOptionalsAreSet or allOptionalNonDefault can be set for each message') if (typeof o !== 'object' || o === null) return new Error(path + ': object is not an instance of an object or is null') - if (typeof o.fromIndex !== 'number') return new Error(`${path}.fromIndex: is not a number`) - if (opts.fromIndex_CustomCheck && !opts.fromIndex_CustomCheck(o.fromIndex)) return new Error(`${path}.fromIndex: custom check failed`) + const fromIndexErr = OperationsCursorValidate(o.fromIndex, opts.fromIndex_Options, `${path}.fromIndex`) + if (fromIndexErr !== null) return fromIndexErr + if (!Array.isArray(o.operations)) return new Error(`${path}.operations: is not an array`) for (let index = 0; index < o.operations.length; index++) { @@ -4024,8 +4054,9 @@ export const UserOperationsValidate = (o?: UserOperations, opts: UserOperationsO } if (opts.operations_CustomCheck && !opts.operations_CustomCheck(o.operations)) return new Error(`${path}.operations: custom check failed`) - if (typeof o.toIndex !== 'number') return new Error(`${path}.toIndex: is not a number`) - if (opts.toIndex_CustomCheck && !opts.toIndex_CustomCheck(o.toIndex)) return new Error(`${path}.toIndex: custom check failed`) + const toIndexErr = OperationsCursorValidate(o.toIndex, opts.toIndex_Options, `${path}.toIndex`) + if (toIndexErr !== null) return toIndexErr + return null } diff --git a/proto/service/structs.proto b/proto/service/structs.proto index 2b976afe..ae5b673b 100644 --- a/proto/service/structs.proto +++ b/proto/service/structs.proto @@ -534,13 +534,19 @@ message UserInfo{ string bridge_url = 11; string nmanage = 12; } + + +message OperationsCursor { + int64 ts = 1; // last timestamp + int64 id = 2; // last serial_id +} message GetUserOperationsRequest{ - int64 latestIncomingInvoice = 1; - int64 latestOutgoingInvoice = 2; - int64 latestIncomingTx = 3; - int64 latestOutgoingTx = 4; - int64 latestIncomingUserToUserPayment = 5; - int64 latestOutgoingUserToUserPayment = 6; + OperationsCursor latestIncomingInvoice = 1; + OperationsCursor latestOutgoingInvoice = 2; + OperationsCursor latestIncomingTx = 3; + OperationsCursor latestOutgoingTx = 4; + OperationsCursor latestIncomingUserToUserPayment = 5; + OperationsCursor latestOutgoingUserToUserPayment = 6; int64 max_size = 7; } enum UserOperationType { @@ -566,8 +572,8 @@ message UserOperation { bool internal = 11; } message UserOperations { - int64 fromIndex=1; - int64 toIndex=2; + OperationsCursor fromIndex=1; + OperationsCursor toIndex=2; repeated UserOperation operations=3; } message GetUserOperationsResponse{ diff --git a/src/services/main/liquidityProvider.ts b/src/services/main/liquidityProvider.ts index 042cef29..01d60b94 100644 --- a/src/services/main/liquidityProvider.ts +++ b/src/services/main/liquidityProvider.ts @@ -227,9 +227,9 @@ export class LiquidityProvider { throw new Error("liquidity provider is not ready yet") } const res = await this.client.GetUserOperations({ - latestIncomingInvoice: 0, latestOutgoingInvoice: 0, - latestIncomingTx: 0, latestOutgoingTx: 0, latestIncomingUserToUserPayment: 0, - latestOutgoingUserToUserPayment: 0, max_size: 200 + latestIncomingInvoice: { ts: 0, id: 0 }, latestOutgoingInvoice: { ts: 0, id: 0 }, + latestIncomingTx: { ts: 0, id: 0 }, latestOutgoingTx: { ts: 0, id: 0 }, latestIncomingUserToUserPayment: { ts: 0, id: 0 }, + latestOutgoingUserToUserPayment: { ts: 0, id: 0 }, max_size: 200 }) if (res.status === 'ERROR') { this.log("error getting operations", res.reason) diff --git a/src/services/main/paymentManager.ts b/src/services/main/paymentManager.ts index 983468ea..ff0f04fb 100644 --- a/src/services/main/paymentManager.ts +++ b/src/services/main/paymentManager.ts @@ -626,14 +626,15 @@ export default class { mapOperations(operations: UserOperationInfo[], type: Types.UserOperationType, inbound: boolean): Types.UserOperations { if (operations.length === 0) { return { - fromIndex: 0, - toIndex: 0, + fromIndex: { ts: 0, id: 0 }, + toIndex: { ts: 0, id: 0 }, operations: [] } } return { - toIndex: operations[0].serial_id, - fromIndex: operations[operations.length - 1].serial_id, + // We fetch in ascending order + toIndex: { ts: operations.at(-1)!.paid_at_unix, id: operations.at(-1)!.serial_id } , + fromIndex: { ts: operations[0].paid_at_unix, id: operations[0]!.serial_id }, operations: operations.map((o: UserOperationInfo): Types.UserOperation => { let identifier = ""; if (o.invoice) { @@ -687,12 +688,12 @@ export default class { throw new Error("user is banned, cannot retrieve operations") } const [outgoingInvoices, outgoingTransactions, incomingInvoices, incomingTransactions, incomingUserToUser, outgoingUserToUser] = await Promise.all([ - this.storage.paymentStorage.GetUserInvoicePayments(userId, req.latestOutgoingInvoice, req.max_size), // - this.storage.paymentStorage.GetUserTransactionPayments(userId, req.latestOutgoingTx, req.max_size), - this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(userId, req.latestIncomingInvoice, req.max_size), - this.storage.paymentStorage.GetUserReceivingTransactions(userId, req.latestIncomingTx, req.max_size), - this.storage.paymentStorage.GetUserToUserReceivedPayments(userId, req.latestIncomingUserToUserPayment, req.max_size), - this.storage.paymentStorage.GetUserToUserSentPayments(userId, req.latestOutgoingUserToUserPayment, req.max_size) + this.storage.paymentStorage.GetUserInvoicePayments(userId, req.latestOutgoingInvoice.id, req.max_size), // + this.storage.paymentStorage.GetUserTransactionPayments(userId, req.latestOutgoingTx.id, req.max_size), + this.storage.paymentStorage.GetUserInvoicesFlaggedAsPaid(user.serial_id, req.latestIncomingInvoice.id, req.latestIncomingInvoice.ts, req.max_size), + this.storage.paymentStorage.GetUserReceivingTransactions(userId, req.latestIncomingTx.id, req.max_size), + this.storage.paymentStorage.GetUserToUserReceivedPayments(userId, req.latestIncomingUserToUserPayment.id, req.max_size), + this.storage.paymentStorage.GetUserToUserSentPayments(userId, req.latestOutgoingUserToUserPayment.id, req.max_size) ]) return { latestIncomingInvoiceOperations: this.mapOperations(incomingInvoices, Types.UserOperationType.INCOMING_INVOICE, true), diff --git a/src/services/storage/db/serializationHelpers.ts b/src/services/storage/db/serializationHelpers.ts index 95b6c0f6..21dfbd2f 100644 --- a/src/services/storage/db/serializationHelpers.ts +++ b/src/services/storage/db/serializationHelpers.ts @@ -1,4 +1,4 @@ -import { FindOperator, LessThan, MoreThan, LessThanOrEqual, MoreThanOrEqual, Equal, Like, ILike, Between, In, Any, IsNull, Not, FindOptionsWhere } from 'typeorm'; +import { FindOperator, LessThan, MoreThan, LessThanOrEqual, MoreThanOrEqual, Equal, Like, ILike, Between, In, Any, IsNull, Not, FindOptionsWhere, And } from 'typeorm'; export type WhereCondition = FindOptionsWhere | FindOptionsWhere[] type SerializedFindOperator = { @@ -11,7 +11,7 @@ export function serializeFindOperator(operator: FindOperator): SerializedFi return { _type: 'FindOperator', type: operator['type'], - value: operator['value'], + value: Array.isArray(operator['value']) ? operator["value"].map(serializeFindOperator) : operator["value"], }; } @@ -41,6 +41,11 @@ export function deserializeFindOperator(serialized: SerializedFindOperator): Fin return IsNull(); case 'not': return Not(deserializeFindOperator(serialized.value)); + case 'and': + if (serialized.value.length !== 2) { + throw new Error("Typeorm and operator with more than 2 arguments not supported"); + } + return And(deserializeFindOperator(serialized.value[0]), deserializeFindOperator(serialized.value[1])); default: throw new Error(`Unknown FindOperator type: ${serialized.type}`); } diff --git a/src/services/storage/entity/UserReceivingInvoice.ts b/src/services/storage/entity/UserReceivingInvoice.ts index 87fd8908..f8fb7001 100644 --- a/src/services/storage/entity/UserReceivingInvoice.ts +++ b/src/services/storage/entity/UserReceivingInvoice.ts @@ -9,6 +9,7 @@ export type ZapInfo = { description: string } @Entity() +@Index("recv_invoice_paid_serial", ["user.serial_id", "paid_at_unix", "serial_id"], { where: "paid_at_unix > 0" }) export class UserReceivingInvoice { @PrimaryGeneratedColumn() diff --git a/src/services/storage/migrations/1753106599604-old_something_leftover.ts b/src/services/storage/migrations/1753106599604-old_something_leftover.ts new file mode 100644 index 00000000..6c97f8a3 --- /dev/null +++ b/src/services/storage/migrations/1753106599604-old_something_leftover.ts @@ -0,0 +1,27 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + + +// Remove the old left over column "something" from user_transaction_payment +export class OldSomethingLeftover1753106599604 implements MigrationInterface { + name = 'OldSomethingLeftover1753106599604' + + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "user_transaction_unique"`); + await queryRunner.query(`CREATE TABLE "temporary_user_transaction_payment" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "address" varchar NOT NULL, "tx_hash" varchar NOT NULL, "output_index" integer NOT NULL, "paid_amount" integer NOT NULL, "chain_fees" integer NOT NULL, "service_fees" integer NOT NULL, "paid_at_unix" integer NOT NULL, "internal" boolean NOT NULL DEFAULT (0), "confs" integer NOT NULL DEFAULT (0), "broadcast_height" integer NOT NULL DEFAULT (0), "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')), "userSerialId" integer, "linkedApplicationSerialId" integer, CONSTRAINT "FK_75abdd29270979e901da0dba7b9" FOREIGN KEY ("linkedApplicationSerialId") REFERENCES "application" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_6f24c901b4103f7146eb615a5db" FOREIGN KEY ("userSerialId") REFERENCES "user" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION)`); + await queryRunner.query(`INSERT INTO "temporary_user_transaction_payment"("serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId") SELECT "serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId" FROM "user_transaction_payment"`); + await queryRunner.query(`DROP TABLE "user_transaction_payment"`); + await queryRunner.query(`ALTER TABLE "temporary_user_transaction_payment" RENAME TO "user_transaction_payment"`); + await queryRunner.query(`CREATE UNIQUE INDEX "user_transaction_unique" ON "user_transaction_payment" ("tx_hash", "output_index") `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "user_transaction_unique"`); + await queryRunner.query(`ALTER TABLE "user_transaction_payment" RENAME TO "temporary_user_transaction_payment"`); + await queryRunner.query(`CREATE TABLE "user_transaction_payment" ("serial_id" integer PRIMARY KEY AUTOINCREMENT NOT NULL, "address" varchar NOT NULL, "tx_hash" varchar NOT NULL, "output_index" integer NOT NULL, "paid_amount" integer NOT NULL, "chain_fees" integer NOT NULL, "service_fees" integer NOT NULL, "paid_at_unix" integer NOT NULL, "internal" boolean NOT NULL DEFAULT (0), "confs" integer NOT NULL DEFAULT (0), "broadcast_height" integer NOT NULL DEFAULT (0), "something" integer NOT NULL DEFAULT (0), "created_at" datetime NOT NULL DEFAULT (datetime('now')), "updated_at" datetime NOT NULL DEFAULT (datetime('now')), "userSerialId" integer, "linkedApplicationSerialId" integer, CONSTRAINT "FK_75abdd29270979e901da0dba7b9" FOREIGN KEY ("linkedApplicationSerialId") REFERENCES "application" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION, CONSTRAINT "FK_6f24c901b4103f7146eb615a5db" FOREIGN KEY ("userSerialId") REFERENCES "user" ("serial_id") ON DELETE NO ACTION ON UPDATE NO ACTION)`); + await queryRunner.query(`INSERT INTO "user_transaction_payment"("serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId") SELECT "serial_id", "address", "tx_hash", "output_index", "paid_amount", "chain_fees", "service_fees", "paid_at_unix", "internal", "confs", "broadcast_height", "created_at", "updated_at", "userSerialId", "linkedApplicationSerialId" FROM "temporary_user_transaction_payment"`); + await queryRunner.query(`DROP TABLE "temporary_user_transaction_payment"`); + await queryRunner.query(`CREATE UNIQUE INDEX "user_transaction_unique" ON "user_transaction_payment" ("tx_hash", "output_index") `); + } + +} diff --git a/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.ts b/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.ts new file mode 100644 index 00000000..8f9c3875 --- /dev/null +++ b/src/services/storage/migrations/1753109184611-user_receiving_invoice_idx.ts @@ -0,0 +1,14 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class UserReceivingInvoiceIdx1753109184611 implements MigrationInterface { + name = 'UserReceivingInvoiceIdx1753109184611' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX "recv_invoice_paid_serial" ON "user_receiving_invoice" ("userSerialId", "paid_at_unix", "serial_id") WHERE paid_at_unix > 0`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "recv_invoice_paid_serial"`); + } + +} diff --git a/src/services/storage/migrations/runner.ts b/src/services/storage/migrations/runner.ts index ec68c749..2786110d 100644 --- a/src/services/storage/migrations/runner.ts +++ b/src/services/storage/migrations/runner.ts @@ -20,9 +20,11 @@ import { ChannelEvents1750777346411 } from './1750777346411-channel_events.js' import { ManagementGrant1751307732346 } from './1751307732346-management_grant.js' import { ManagementGrantBanned1751989251513 } from './1751989251513-management_grant_banned.js' import { InvoiceCallbackUrls1752425992291 } from './1752425992291-invoice_callback_urls.js' +import { OldSomethingLeftover1753106599604 } from './1753106599604-old_something_leftover.js' +import { UserReceivingInvoiceIdx1753109184611 } from './1753109184611-user_receiving_invoice_idx.js' export const allMigrations = [Initial1703170309875, LspOrder1718387847693, LiquidityProvider1719335699480, LndNodeInfo1720187506189, TrackedProvider1720814323679, CreateInviteTokenTable1721751414878, PaymentIndex1721760297610, DebitAccess1726496225078, DebitAccessFixes1726685229264, - DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291] + DebitToPub1727105758354, UserCbUrl1727112281043, UserOffer1733502626042, ManagementGrant1751307732346, ManagementGrantBanned1751989251513, InvoiceCallbackUrls1752425992291, OldSomethingLeftover1753106599604, UserReceivingInvoiceIdx1753109184611] export const allMetricsMigrations = [LndMetrics1703170330183, ChannelRouting1709316653538, HtlcCount1724266887195, BalanceEvents1724860966825, RootOps1732566440447, RootOpsTime1745428134124, ChannelEvents1750777346411] /* export const TypeOrmMigrationRunner = async (log: PubLogger, storageManager: Storage, settings: DbSettings, arg: string | undefined): Promise => { await connectAndMigrate(log, storageManager, allMigrations, allMetricsMigrations) diff --git a/src/services/storage/paymentStorage.ts b/src/services/storage/paymentStorage.ts index e337f9ea..f651923c 100644 --- a/src/services/storage/paymentStorage.ts +++ b/src/services/storage/paymentStorage.ts @@ -1,5 +1,5 @@ import crypto from 'crypto'; -import { Between, FindOperator, IsNull, LessThanOrEqual, MoreThan, MoreThanOrEqual, Not } from "typeorm" +import { And, Between, Equal, FindOperator, IsNull, LessThanOrEqual, MoreThan, MoreThanOrEqual } from "typeorm" import { User } from './entity/User.js'; import { UserTransactionPayment } from './entity/UserTransactionPayment.js'; import { EphemeralKeyType, UserEphemeralKey } from './entity/UserEphemeralKey.js'; @@ -42,11 +42,11 @@ export default class { return this.dbs.Find('AddressReceivingTransaction', { where: { user_address: { user: { user_id: userId } }, - serial_id: MoreThanOrEqual(fromIndex), + serial_id: MoreThan(fromIndex), paid_at_unix: MoreThan(0), }, order: { - paid_at_unix: 'DESC' + paid_at_unix: 'ASC' }, take }, txId) @@ -73,20 +73,42 @@ export default class { return this.dbs.Update('UserReceivingInvoice', invoice.serial_id, i, txId) } - GetUserInvoicesFlaggedAsPaid(userId: string, fromIndex: number, take = 50, txId?: string): Promise { - return this.dbs.Find('UserReceivingInvoice', { - where: { - user: { - user_id: userId + async GetUserInvoicesFlaggedAsPaid(userSerialId: number, fromIndex: number, fromPaidTimestamp: number, take = 50, txId?: string): Promise { + let items: UserReceivingInvoice[] = []; + if (fromPaidTimestamp > 0) { + // First fetch same paid_at_unix, higher serial_id + const firstBatch = await this.dbs.Find('UserReceivingInvoice', { + where: { + user: { serial_id: userSerialId }, + paid_at_unix: And(MoreThan(0), Equal(fromPaidTimestamp)), + serial_id: MoreThan(fromIndex) }, - serial_id: MoreThanOrEqual(fromIndex), - paid_at_unix: MoreThan(0), - }, - order: { - paid_at_unix: 'DESC' - }, - take - }, txId) + order: { + paid_at_unix: 'ASC', + serial_id: 'ASC' + }, + take + }, txId); + items.push(...firstBatch); + } + + const needMore = take - items.length + // If need more, fetch higher paid_at_unix + if (needMore > 0) { + const secondBatch = await this.dbs.Find('UserReceivingInvoice', { + where: { + user: { serial_id: userSerialId }, + paid_at_unix: And(MoreThan(0), MoreThan(fromPaidTimestamp)), + }, + order: { + paid_at_unix: 'ASC', + serial_id: 'ASC' + }, + take: needMore + }, txId) + items.push(...secondBatch) + } + return items } async AddUserInvoice(user: User, invoice: string, options: InboundOptionals = { expiry: defaultInvoiceExpiry }, providerDestination?: string, txId?: string): Promise { @@ -184,11 +206,11 @@ export default class { user: { user_id: userId }, - serial_id: MoreThanOrEqual(fromIndex), + serial_id: MoreThan(fromIndex), paid_at_unix: MoreThan(-1), }, order: { - paid_at_unix: 'DESC' + paid_at_unix: 'ASC' }, take }, txId) @@ -232,11 +254,11 @@ export default class { user: { user_id: userId }, - serial_id: MoreThanOrEqual(fromIndex), + serial_id: MoreThan(fromIndex), paid_at_unix: MoreThan(0), }, order: { - paid_at_unix: 'DESC' + paid_at_unix: 'ASC' }, take }, txId) @@ -301,11 +323,11 @@ export default class { to_user: { user_id: userId }, - serial_id: MoreThanOrEqual(fromIndex), + serial_id: MoreThan(fromIndex), paid_at_unix: MoreThan(0), }, order: { - paid_at_unix: 'DESC' + paid_at_unix: 'ASC' }, take }, txId) @@ -318,11 +340,11 @@ export default class { from_user: { user_id: userId }, - serial_id: MoreThanOrEqual(fromIndex), + serial_id: MoreThan(fromIndex), paid_at_unix: MoreThan(0), }, order: { - paid_at_unix: 'DESC' + paid_at_unix: 'ASC' }, take }, txId)