feat(database): recalculate all balances migration

database/migration/index.ts

@@ -46,6 +46,9 @@ const run = async (command: string) => {
     user: CONFIG.DB_USER,
     password: CONFIG.DB_PASSWORD,
     database: CONFIG.DB_DATABASE,
+    supportBigNumbers: true,
+    bigNumberStrings: true,
+    multipleStatements: true
   })
   const migration = new Migration({
     conn: pool,

database/migration/migrations/0102-recalculate_balances.ts

@@ -0,0 +1,74 @@
+import { DecayCalculationType, GradidoUnit } from 'shared'
+import { calculateDecay } from 'shared-native'
+
+export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  // Find all users & loop over them
+  const runningRequests: Promise<Array<any>>[] = []
+  const users = await queryFn('SELECT id FROM users')
+  let countDiffs = 0
+  for (let u = 0; u < users.length; u++) {
+    process.stdout.write(`Recalculating balances for user ${u}/${users.length}\r`)
+    // find all transactions for a user
+    const transactions = await queryFn(`
+       SELECT id, amount_gdd4, balance_gdd4, decay_gdd4, balance_date
+       FROM transactions
+       WHERE user_id = ${users[u].id}
+       ORDER BY balance_date ASC
+       ;
+    `)
+    let previous = null
+    let balance = 0n
+    const transactionsToUpdate: string[] = []
+
+    for (let t = 0; t < transactions.length; t++) {
+      const transaction = transactions[t]
+      const amount = BigInt(transaction.amount_gdd4)
+      let decay = 0n
+      if (balance && previous) {
+        const decayedBalance = calculateDecay(
+          balance,
+          GradidoUnit.effectiveDecayDuration(previous.balance_date, transaction.balance_date)
+            .seconds,
+        )
+        decay = decayedBalance - balance
+        balance = decayedBalance
+      }
+      balance += amount
+      if (BigInt(transaction.balance_gdd4) !== balance) {
+        countDiffs++
+        transactionsToUpdate.push(
+          `UPDATE transactions
+           SET balance_gdd4 = '${balance.toString()}',
+               decay_gdd4 = '${decay.toString()}'
+           WHERE id = ${transaction.id}
+           ;
+        `,
+        )
+      }
+      previous = transaction
+    }
+    if (transactionsToUpdate.length) {
+      runningRequests.push(queryFn(transactionsToUpdate.join('\n')))
+      // await queryFn(transactionsToUpdate.join('\n'))
+    }
+  }
+  runningRequests.push(
+    queryFn(
+      `UPDATE transactions set decay_calculation_type = ${DecayCalculationType.NATIVE_C_FIXED_FACTOR_INTEGER}`,
+    ),
+  )
+  await Promise.all(runningRequests)
+  process.stdout.write(`\n`)
+  // biome-ignore lint/suspicious/noConsole: no logger present
+  console.log(`${countDiffs} updated transactions`)
+}
+
+export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  await queryFn(`
+    UPDATE transactions
+    SET balance_gdd4 = CAST(ROUND(balance_legacy * 10000) AS SIGNED),
+        decay_gdd4 = CAST(ROUND(decay_legacy * 10000) AS SIGNED); `)
+  await queryFn(
+    `UPDATE transactions set decay_calculation_type = ${DecayCalculationType.DECIMAL_JS_FIXED_FACTOR}`,
+  )
+}

dlt-connector/package.json

@@ -10,7 +10,7 @@
     "start": "cross-env TZ=UTC bun run src/index.ts",
     "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify",
     "dev": "cross-env TZ=UTC bun run --watch src/index.ts",
-    "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7",
+    "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7 --use-only-legacy-ledger-anchor-ids",
     "test": "cross-env TZ=UTC bun test",
     "test:debug": "cross-env TZ=UTC bun test --inspect-brk",
     "typecheck": "tsc --noEmit",

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts

@@ -21,6 +21,7 @@ export class Context {
   public communities: Map<string, CommunityContext>
   public cache: KeyPairCacheManager
   public nativeDecayCalculationStartDate: Date
+  public useOnlyLegacyLedgerAnchorIds: boolean = false
   private timeUsed: MonotonicTimer
 
   constructor(
@@ -35,6 +36,7 @@ export class Context {
     this.communities = new Map<string, CommunityContext>()
     this.nativeDecayCalculationStartDate = nativeDecayCalculationStartDate
     this.timeUsed = new MonotonicTimer()
+    this.useOnlyLegacyLedgerAnchorIds = process.argv.includes('--use-only-legacy-ledger-anchor-ids')
   }
 
   static async create(): Promise<Context> {

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts

@@ -2,16 +2,19 @@ import {
   AccountBalances,
   Filter,
   GradidoTransactionBuilder,
+  HieroTransactionId,
   InMemoryBlockchain,
   KeyPairEd25519,
+  LedgerAnchor,
+  LedgerAnchor_Type,
   MemoryBlockPtr,
   MonotonicTimer,
   SearchDirection_DESC,
 } from 'gradido-blockchain-js'
 import { getLogger, Logger } from 'log4js'
 import { LOG4JS_BASE_CATEGORY } from '../../../../config/const'
 import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair'
-import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
+import { HieroTransactionIdString, Uuidv4 } from '../../../../schemas/typeGuard.schema'
 import { Context } from '../../Context'
 import { Balance } from '../../data/Balance'
 import { CommunityContext } from '../../valibot.schema'
@@ -20,22 +23,31 @@ export type IndexType = {
   date: Date
   id: number
 }
+export interface SyncItem {
+  id: number
+  messageId?: HieroTransactionIdString | null | undefined
+}
 export let nanosBalanceForUser = 0
 const lastBalanceOfUserTimeUsed = new MonotonicTimer()
 
-export abstract class AbstractSyncRole<ItemType> {
+export abstract class AbstractSyncRole<ItemType extends SyncItem> {
   private items: ItemType[] = []
   protected lastIndex: IndexType = { date: new Date(0), id: 0 }
   protected logger: Logger
   protected transactionBuilder: GradidoTransactionBuilder
   protected accountBalances: AccountBalances
+  protected legacyAnchorType: LedgerAnchor_Type
 
-  constructor(protected readonly context: Context) {
+  constructor(
+    protected readonly context: Context,
+    legacyAnchorType: LedgerAnchor_Type,
+  ) {
     this.logger = getLogger(
       `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`,
     )
     this.transactionBuilder = new GradidoTransactionBuilder()
     this.accountBalances = new AccountBalances()
+    this.legacyAnchorType = legacyAnchorType
   }
 
   getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 {
@@ -84,6 +96,14 @@ export abstract class AbstractSyncRole<ItemType> {
     return result
   }
 
+  getLedgerAnchor(item: ItemType): LedgerAnchor {
+    if (item.messageId && !this.context.useOnlyLegacyLedgerAnchorIds) {
+      return new LedgerAnchor(new HieroTransactionId(item.messageId))
+    } else {
+      return new LedgerAnchor(item.id, this.legacyAnchorType)
+    }
+  }
+
   logLastBalanceChangingTransactions(
     publicKey: MemoryBlockPtr,
     blockchain: InMemoryBlockchain,

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/CreationsSync.role.ts

@@ -6,7 +6,6 @@ import {
   EncryptedMemo,
   Filter,
   GradidoTransactionBuilder,
-  HieroTransactionId,
   KeyPairEd25519,
   LedgerAnchor,
   MemoryBlockPtr,
@@ -31,7 +30,7 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
 
 export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
   constructor(context: Context) {
-    super(context)
+    super(context, LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID)
     this.accountBalances.reserve(3n)
   }
 
@@ -165,22 +164,11 @@ export class CreationsSyncRole extends AbstractSyncRole<CreationTransactionDb> {
     }
 
     try {
-      let ledgerAnchor: LedgerAnchor | undefined
-      if (item.messageId) {
-        ledgerAnchor = new LedgerAnchor(new HieroTransactionId(item.messageId))
-      } else {
-        ledgerAnchor = new LedgerAnchor(
-          item.id,
-          LedgerAnchor.Type_LEGACY_GRADIDO_DB_CONTRIBUTION_ID,
-        )
-      }
       addToBlockchain(
         this.buildTransaction(item, communityContext, recipientKeyPair, signerKeyPair).build(),
         blockchain,
-        ledgerAnchor,
-        !this.context.isDecayCalculationTypeChanged(item.confirmedAt)
-          ? this.calculateAccountBalances(item, communityContext, recipientPublicKey)
-          : undefined,
+        this.getLedgerAnchor(item),
+        this.calculateAccountBalances(item, communityContext, recipientPublicKey),
       )
     } catch (e) {
       const f = new Filter()

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/DeletedTransactionLinksSync.role.ts

@@ -7,7 +7,6 @@ import {
   GradidoTransactionBuilder,
   GradidoTransfer,
   GradidoUnit,
-  HieroTransactionId,
   KeyPairEd25519,
   LedgerAnchor,
   MemoryBlockPtr,
@@ -31,7 +30,7 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
 
 export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTransactionLinkDb> {
   constructor(context: Context) {
-    super(context)
+    super(context, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID)
     this.accountBalances.reserve(2n)
   }
 
@@ -186,15 +185,6 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
     senderLastBalance.updateLegacyDecay(GradidoUnit.zero(), item.deletedAt)
 
     try {
-      let ledgerAnchor: LedgerAnchor
-      if (item.messageId) {
-        ledgerAnchor = new LedgerAnchor(new HieroTransactionId(item.messageId))
-      } else {
-        ledgerAnchor = new LedgerAnchor(
-          item.id,
-          LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID,
-        )
-      }
       addToBlockchain(
         this.buildTransaction(
           communityContext,
@@ -205,7 +195,7 @@ export class DeletedTransactionLinksSyncRole extends AbstractSyncRole<DeletedTra
           linkFundingPublicKey,
         ).build(),
         blockchain,
-        ledgerAnchor,
+        this.getLedgerAnchor(item),
         this.calculateBalances(
           item,
           deferredTransfer,

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/LocalTransactionsSync.role.ts

@@ -6,7 +6,6 @@ import {
   AuthenticatedEncryption,
   EncryptedMemo,
   GradidoTransactionBuilder,
-  HieroTransactionId,
   KeyPairEd25519,
   LedgerAnchor,
   MemoryBlockPtr,
@@ -31,7 +30,7 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
 
 export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
   constructor(context: Context) {
-    super(context)
+    super(context, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID)
     this.accountBalances.reserve(2n)
   }
 
@@ -181,19 +180,11 @@ export class LocalTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
     }
 
     try {
-      let ledgerAnchor: LedgerAnchor | undefined
-      if (item.messageId) {
-        ledgerAnchor = new LedgerAnchor(new HieroTransactionId(item.messageId))
-      } else {
-        ledgerAnchor = new LedgerAnchor(item.id, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID)
-      }
       addToBlockchain(
         this.buildTransaction(communityContext, item, senderKeyPair, recipientKeyPair).build(),
         blockchain,
-        ledgerAnchor,
-        item.decayCalculationType === DecayCalculationType.DECIMAL_JS_FIXED_FACTOR
-          ? this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey)
-          : undefined,
+        this.getLedgerAnchor(item),
+        this.calculateBalances(item, communityContext, senderPublicKey, recipientPublicKey),
       )
     } catch (e) {
       if (e instanceof NotEnoughGradidoBalanceError) {

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RedeemTransactionLinksSync.role.ts

@@ -10,7 +10,6 @@ import {
   GradidoTransactionBuilder,
   GradidoTransfer,
   GradidoUnit,
-  HieroTransactionId,
   KeyPairEd25519,
   LedgerAnchor,
   MemoryBlockPtr,
@@ -39,7 +38,7 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
 
 export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTransactionLinkDb> {
   constructor(context: Context) {
-    super(context)
+    super(context, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID)
     this.accountBalances.reserve(3n)
   }
 
@@ -226,15 +225,6 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
     }
 
     try {
-      let ledgerAnchor: LedgerAnchor | undefined
-      if (item.messageId) {
-        ledgerAnchor = new LedgerAnchor(new HieroTransactionId(item.messageId))
-      } else {
-        ledgerAnchor = new LedgerAnchor(
-          item.id,
-          LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_LINK_ID,
-        )
-      }
       addToBlockchain(
         this.buildTransaction(
           communityContext,
@@ -244,16 +234,14 @@ export class RedeemTransactionLinksSyncRole extends AbstractSyncRole<RedeemedTra
           recipientKeyPair,
         ).build(),
         blockchain,
-        ledgerAnchor,
-        this.context.isDecayCalculationTypeChanged(item.redeemedAt)
-          ? undefined
-          : this.calculateBalances(
-              item,
-              deferredTransfer,
-              communityContext,
-              senderPublicKey,
-              recipientPublicKey,
-            ),
+        this.getLedgerAnchor(item),
+        this.calculateBalances(
+          item,
+          deferredTransfer,
+          communityContext,
+          senderPublicKey,
+          recipientPublicKey,
+        ),
       )
     } catch (e) {
       throw new BlockchainError(`Error adding ${this.itemTypeName()}`, item, e as Error)

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/RemoteTransactionsSync.role.ts

@@ -32,7 +32,7 @@ import { AbstractSyncRole, IndexType } from './AbstractSync.role'
 
 export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb> {
   constructor(context: Context) {
-    super(context)
+    super(context, LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID)
     this.accountBalances.reserve(1n)
   }
 
@@ -190,10 +190,7 @@ export class RemoteTransactionsSyncRole extends AbstractSyncRole<TransactionDb>
 
   pushToBlockchain(item: TransactionDb): void {
     const { senderUser, recipientUser } = this.getUser(item)
-    const ledgerAnchor = new LedgerAnchor(
-      item.id,
-      LedgerAnchor.Type_LEGACY_GRADIDO_DB_TRANSACTION_ID,
-    )
+    const ledgerAnchor = this.getLedgerAnchor(item)
 
     if (senderUser.communityUuid === recipientUser.communityUuid) {
       throw new Error(