feat(database): recalculate all balances migration

database/migration/index.ts

@@ -46,6 +46,9 @@ const run = async (command: string) => {
     user: CONFIG.DB_USER,
     password: CONFIG.DB_PASSWORD,
     database: CONFIG.DB_DATABASE,
+    supportBigNumbers: true,
+    bigNumberStrings: true,
+    multipleStatements: true
   })
   const migration = new Migration({
     conn: pool,

database/migration/migrations/0102-recalculate_balances.ts

@@ -0,0 +1,83 @@
+import { DecayCalculationType, GradidoUnit } from 'shared'
+import { calculateDecay } from 'shared-native'
+
+export async function upgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  // Find all users & loop over them
+  const runningRequests: Promise<Array<any>>[] = []
+  const users = await queryFn('SELECT id FROM users')
+  let countDiffs = 0
+  for (let u = 0; u < users.length; u++) {
+    process.stdout.write(`Recalculating balances for user ${u}/${users.length}\r`)
+    // find all transactions for a user
+    const transactions = await queryFn(
+      `
+       SELECT id, amount_gdd4, balance_gdd4, balance_legacy, decay_gdd4, balance_date
+       FROM transactions
+       WHERE user_id = ?
+       ORDER BY balance_date ASC, id ASC
+       ;
+    `,
+      [users[u].id],
+    )
+
+    let previous = null
+    let balance = 0n
+    const transactionsToUpdate: string[] = []
+
+    for (let t = 0; t < transactions.length; t++) {
+      const transaction = transactions[t]
+      const amount = BigInt(transaction.amount_gdd4)
+      let decay = 0n
+      if (balance > 0n && previous) {
+        const decayedBalance = calculateDecay(
+          balance,
+          GradidoUnit.effectiveDecayDuration(previous.balance_date, transaction.balance_date)
+            .seconds,
+        )
+        decay = decayedBalance - balance
+        balance = decayedBalance
+      } else if (balance <= 0n) {
+        if (balance < 0) {
+          // biome-ignore lint/suspicious/noConsole: no logger in migration
+          console.warn(
+            `set negative balance: ${balance.toString()} for transaction: ${transaction.id} to zero.`,
+          )
+        }
+        balance = 0n
+      }
+      balance += amount
+      if (BigInt(transaction.balance_gdd4) !== balance) {
+        countDiffs++
+        transactionsToUpdate.push(
+          `UPDATE transactions
+           SET balance_gdd4 = '${balance.toString()}',
+               decay_gdd4 = '${decay.toString()}'
+           WHERE id = ${transaction.id}
+           ;
+        `,
+        )
+        previous = transaction
+      }
+    }
+    if (transactionsToUpdate.length) {
+      runningRequests.push(queryFn(transactionsToUpdate.join('\n')))
+    }
+  }
+  await Promise.all(runningRequests)
+  await queryFn(`UPDATE transactions set decay_calculation_type = ?`, [
+    DecayCalculationType.NATIVE_C_FIXED_FACTOR_INTEGER,
+  ])
+  process.stdout.write(`\n`)
+  // biome-ignore lint/suspicious/noConsole: no logger present
+  console.log(`${countDiffs} updated transactions`)
+}
+
+export async function downgrade(queryFn: (query: string, values?: any[]) => Promise<Array<any>>) {
+  await queryFn(`
+    UPDATE transactions
+    SET balance_gdd4 = CAST(ROUND(balance_legacy * 10000) AS SIGNED),
+        decay_gdd4 = CAST(ROUND(decay_legacy * 10000) AS SIGNED); `)
+  await queryFn(
+    `UPDATE transactions set decay_calculation_type = ${DecayCalculationType.DECIMAL_JS_FIXED_FACTOR}`,
+  )
+}

database/migration/scripts/removeCrossGroupTransactions.ts

@@ -39,7 +39,13 @@ function calculateBalance(
     }
   } else {
     // update balance with decay
-    const lastTransactionBalance = GradidoUnit.fromGradidoCent(lastTransaction.balance)
+    let lastTransactionBalance = GradidoUnit.fromGradidoCent(lastTransaction.balance)
+    if (lastTransactionBalance.gddCent <= 0) {
+      return {
+        balance: currentTransaction.amount,
+        decay: 0n
+      }
+    }
     const decay = lastTransactionBalance.calculateDecay(lastTransaction.balanceDate, currentTransaction.balanceDate)
     const newBalance = decay.balance.add(GradidoUnit.fromGradidoCent(currentTransaction.amount))
 

dlt-connector/package.json

@@ -10,7 +10,7 @@
     "start": "cross-env TZ=UTC bun run src/index.ts",
     "build": "bun build src/index.ts --outdir=build --target=bun --external=gradido-blockchain-js --minify",
     "dev": "cross-env TZ=UTC bun run --watch src/index.ts",
-    "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7",
+    "migrate": "cross-env TZ=UTC bun src/migrations/db-v2.7.0_to_blockchain-v3.7 --use-only-legacy-ledger-anchor-ids",
     "test": "cross-env TZ=UTC bun test",
     "test:debug": "cross-env TZ=UTC bun test --inspect-brk",
     "typecheck": "tsc --noEmit",
@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "cross-env": "^7.0.3",
-    "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#61dabaaca680dedbe6911482d5562f103497d86a"
+    "gradido-blockchain-js": "git+https://github.com/gradido/gradido-blockchain-js#08c8dab30103ba3027a2d056df3d1467ed153edb"
   },
   "devDependencies": {
     "@biomejs/biome": "2.0.0",
@@ -33,7 +33,6 @@
     "@types/uuid": "^8.3.4",
     "adm-zip": "^0.5.16",
     "async-mutex": "^0.5.0",
-    "decimal.js-light": "^2.5.1",
     "dotenv": "^10.0.0",
     "drizzle-orm": "^0.44.7",
     "elysia": "1.3.8",

dlt-connector/src/config/schema.ts

@@ -84,7 +84,7 @@ export const configSchema = v.object({
       v.string('The version of the DLT node server, for example: 0.9.0'),
       v.regex(/^\d+\.\d+\.\d+(.\d+)?$/),
     ),
-    '0.9.8.1',
+    '0.9.8.5',
   ),
   DLT_GRADIDO_NODE_SERVER_HOME_FOLDER: v.optional(
     v.string('The home folder for the gradido dlt node server'),

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/Context.ts

@@ -21,6 +21,7 @@ export class Context {
   public communities: Map<string, CommunityContext>
   public cache: KeyPairCacheManager
   public nativeDecayCalculationStartDate: Date
+  public useOnlyLegacyLedgerAnchorIds: boolean = false
   private timeUsed: MonotonicTimer
 
   constructor(
@@ -35,6 +36,7 @@ export class Context {
     this.communities = new Map<string, CommunityContext>()
     this.nativeDecayCalculationStartDate = nativeDecayCalculationStartDate
     this.timeUsed = new MonotonicTimer()
+    this.useOnlyLegacyLedgerAnchorIds = process.argv.includes('--use-only-legacy-ledger-anchor-ids')
   }
 
   static async create(): Promise<Context> {

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/data/Balance.ts

@@ -1,13 +1,5 @@
-import Decimal from 'decimal.js-light'
 import { AccountBalance, GradidoUnit, MemoryBlockPtr } from 'gradido-blockchain-js'
-import { Amount } from '../../../schemas/typeGuard.schema'
 import { NegativeBalanceError } from '../errors'
-import { legacyCalculateDecay } from '../utils'
-
-Decimal.set({
-  precision: 25,
-  rounding: Decimal.ROUND_HALF_UP,
-})
 
 export class Balance {
   private balance: GradidoUnit
@@ -40,62 +32,6 @@ export class Balance {
     return this.date
   }
 
-  updateLegacyDecay(amount: GradidoUnit, date: Date, dbBalance?: Amount) {
-    // make sure to copy instead of referencing
-    const previousBalanceString = this.balance.toString()
-    const previousDate = new Date(this.date.getTime())
-
-    if (this.balance.equal(GradidoUnit.zero())) {
-      this.balance = amount
-      this.date = date
-    } else {
-      const decayedBalance = legacyCalculateDecay(
-        new Decimal(this.balance.toString()),
-        this.date,
-        date,
-      ).toDecimalPlaces(4, Decimal.ROUND_CEIL)
-      /*
-        if (!decayedBalance.equals(this.balance.toString())) {
-        console.log({
-          input: this.balance.toString(4),
-          decayed: decayedBalance.toString(),
-          amount: amount.toString(4),
-          date
-        })
-        // throw new Error("Test Exit")
-      }
-      // */
-
-      let newBalance = decayedBalance.add(new Decimal(amount.toString()))
-      // reduce small deviations between db decay and balance calculation with 25 decimal places vs gradido-blockchain with 4
-      if (
-        dbBalance &&
-        new Decimal(newBalance.toString()).minus(dbBalance).abs() < new Decimal(0.01)
-      ) {
-        newBalance = new Decimal(dbBalance)
-      }
-      this.balance = GradidoUnit.fromString(newBalance.toString())
-      this.date = date
-    }
-    if (this.balance.lt(GradidoUnit.zero())) {
-      if (this.balance.lt(GradidoUnit.fromGradidoCent(100n).negated())) {
-        const previousDecayedBalance = legacyCalculateDecay(
-          new Decimal(previousBalanceString),
-          previousDate,
-          date,
-        )
-        throw new NegativeBalanceError(
-          `negative Gradido amount detected in Balance.updateLegacyDecay`,
-          previousBalanceString,
-          amount.toString(),
-          previousDecayedBalance.toString(),
-        )
-      } else {
-        this.balance = GradidoUnit.zero()
-      }
-    }
-  }
-
   update(amount: GradidoUnit, date: Date) {
     const previousBalance = GradidoUnit.fromGradidoCent(this.balance.getGradidoCent())
     const previousDate = new Date(this.date.getTime())

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/drizzle.schema.ts

@@ -3,7 +3,6 @@ import {
   bigint,
   char,
   datetime,
-  decimal,
   index,
   int,
   mysqlTable,
@@ -78,7 +77,6 @@ export const transactionsTable = mysqlTable(
     transactionLinkId: int('transaction_link_id').default(sql`NULL`),
     amount: bigint('amount_gdd4', { mode: 'bigint' }).default(sql`NULL`),
     balance: bigint('balance_gdd4', { mode: 'bigint' }).default(sql`NULL`),
-    balanceFull: decimal('balance_legacy', { precision: 40, scale: 20 }).default(sql`NULL`),
     balanceDate: datetime('balance_date', { mode: 'string', fsp: 3 })
       .default(sql`current_timestamp(3)`)
       .notNull(),

dlt-connector/src/migrations/db-v2.7.0_to_blockchain-v3.7/interaction/syncDbWithBlockchain/AbstractSync.role.ts

@@ -2,16 +2,19 @@ import {
   AccountBalances,
   Filter,
   GradidoTransactionBuilder,
+  HieroTransactionId,
   InMemoryBlockchain,
   KeyPairEd25519,
+  LedgerAnchor,
+  LedgerAnchor_Type,
   MemoryBlockPtr,
   MonotonicTimer,
   SearchDirection_DESC,
 } from 'gradido-blockchain-js'
 import { getLogger, Logger } from 'log4js'
 import { LOG4JS_BASE_CATEGORY } from '../../../../config/const'
 import { deriveFromKeyPairAndIndex, deriveFromKeyPairAndUuid } from '../../../../data/deriveKeyPair'
-import { Uuidv4 } from '../../../../schemas/typeGuard.schema'
+import { HieroTransactionIdString, Uuidv4 } from '../../../../schemas/typeGuard.schema'
 import { Context } from '../../Context'
 import { Balance } from '../../data/Balance'
 import { CommunityContext } from '../../valibot.schema'
@@ -20,22 +23,31 @@ export type IndexType = {
   date: Date
   id: number
 }
+export interface SyncItem {
+  id: number
+  messageId?: HieroTransactionIdString | null | undefined
+}
 export let nanosBalanceForUser = 0
 const lastBalanceOfUserTimeUsed = new MonotonicTimer()
 
-export abstract class AbstractSyncRole<ItemType> {
+export abstract class AbstractSyncRole<ItemType extends SyncItem> {
   private items: ItemType[] = []
   protected lastIndex: IndexType = { date: new Date(0), id: 0 }
   protected logger: Logger
   protected transactionBuilder: GradidoTransactionBuilder
   protected accountBalances: AccountBalances
+  protected legacyAnchorType: LedgerAnchor_Type
 
-  constructor(protected readonly context: Context) {
+  constructor(
+    protected readonly context: Context,
+    legacyAnchorType: LedgerAnchor_Type,
+  ) {
     this.logger = getLogger(
       `${LOG4JS_BASE_CATEGORY}.migrations.db-v2.7.0_to_blockchain-v3.5.interaction.syncDbWithBlockchain`,
     )
     this.transactionBuilder = new GradidoTransactionBuilder()
     this.accountBalances = new AccountBalances()
+    this.legacyAnchorType = legacyAnchorType
   }
 
   getAccountKeyPair(communityContext: CommunityContext, gradidoId: Uuidv4): KeyPairEd25519 {
@@ -84,6 +96,14 @@ export abstract class AbstractSyncRole<ItemType> {
     return result
   }
 
+  getLedgerAnchor(item: ItemType): LedgerAnchor {
+    if (item.messageId && !this.context.useOnlyLegacyLedgerAnchorIds) {
+      return new LedgerAnchor(new HieroTransactionId(item.messageId))
+    } else {
+      return new LedgerAnchor(item.id, this.legacyAnchorType)
+    }
+  }
+
   logLastBalanceChangingTransactions(
     publicKey: MemoryBlockPtr,
     blockchain: InMemoryBlockchain,