add balance to accounts

close #11

src/data_handler.ts

@@ -51,29 +51,39 @@ export class DataHandler {
   }
 
   async processNewData(newData: NewData, ctx: Ctx) {
-
-    // Process population history by adding the value
-    // of the last point in database.
-    if (newData.populationHistories) {
-      const lastHistory = await ctx.store.findOneOrFail(PopulationHistory, {
-        where: {},
-        order: { blockNumber: 'DESC' }
-      });
-
-      this.data.populationHistories = newData.populationHistories.map(history => (new PopulationHistory({
-        activeAccountCount: history.activeAccountCount + lastHistory.activeAccountCount,
-        memberCount: history.memberCount + lastHistory.memberCount,
-        smithCount: history.smithCount + lastHistory.smithCount,
-        blockNumber: history.blockNumber,
-        id: `population-${history.blockNumber}`,
-      })));
-    }
-
-    // Process accounts
+    // Process accounts first to ensure they exist
     for (const accountevt of newData.accounts) {
-      const newAccount = new Account({ id: accountevt.address, createdOn: accountevt.blockNumber, isActive: true });
+      const newAccount = await this.getOrCreateAccount(ctx, accountevt.address);
       this.data.accounts.set(accountevt.address, newAccount);
-      ctx.log.info(`Added account ${accountevt}`);
+      // Store immediately to ensure it exists for future references
+      await ctx.store.upsert([newAccount]);
+      ctx.log.info(`Added account ${accountevt.address} at block ${accountevt.blockNumber}`);
+    }
+
+    // Process identities created
+    for (const identity of newData.identitiesCreated) {
+      // Create account if it doesn't exist yet
+      const account = await this.getOrCreateAccount(ctx, identity.accountId);
+      // Store account immediately if it's new
+      if (!this.data.accounts.has(identity.accountId)) {
+        await ctx.store.upsert([account]);
+        this.data.accounts.set(identity.accountId, account);
+      }
+
+      const newIdentity = new Identity({
+        id: identity.event.id,
+        index: identity.index,
+        name: identity.event.id,
+        status: IdentityStatus.Unconfirmed,
+        account,
+        isMember: false,
+        lastChangeOn: identity.blockNumber,
+        createdOn: identity.blockNumber,
+        createdIn: await ctx.store.getOrFail(Event, identity.event.id),
+        expireOn: identity.expireOn,
+      });
+      this.data.identities.set(identity.index, newIdentity);
+      await ctx.store.upsert([newIdentity]);
     }
 
     // Process killed accounts
@@ -86,8 +96,6 @@ export class DataHandler {
 
     // Process transfers
     for (const transfer of newData.transfers) {
-      //   const fromAccount = await this.getOrCreateAccount(ctx, transfer.from);
-      //   const toAccount = await this.getOrCreateAccount(ctx, transfer.to);
       ctx.log.info(
         `New transaction: ${transfer.from} to ${transfer.to} of ${transfer.amount} tokens`
       );
@@ -104,8 +112,55 @@ export class DataHandler {
         amount: transfer.amount,
       });
       this.data.transfers.set(transfer.id, newTransfer);
+
+      // Update balances
+      await this.updateAccountBalance(ctx, transfer.from, -transfer.amount);
+      await this.updateAccountBalance(ctx, transfer.to, transfer.amount);
     }
 
+    // Process deposits
+    for (const deposit of newData.deposits) {
+      ctx.log.info(
+        `New deposit: ${deposit.address} received ${deposit.amount} tokens`
+      );
+      await this.updateAccountBalance(ctx, deposit.address, deposit.amount);
+    }
+
+    // Process withdraws
+    for (const withdraw of newData.withdraws) {
+      ctx.log.info(
+        `New withdraw: ${withdraw.address} withdrew ${withdraw.amount} tokens`
+      );
+      await this.updateAccountBalance(ctx, withdraw.address, -withdraw.amount);
+    }
+
+    // Process reserved amounts
+    for (const reserved of newData.reserved) {
+      ctx.log.info(
+        `New reserve: ${reserved.address} reserved ${reserved.amount} tokens`
+      );
+      await this.updateAccountBalance(ctx, reserved.address, -reserved.amount);
+    }
+
+    // Process unreserved amounts
+    for (const unreserved of newData.unreserved) {
+      ctx.log.info(
+        `New unreserve: ${unreserved.address} unreserved ${unreserved.amount} tokens`
+      );
+      await this.updateAccountBalance(ctx, unreserved.address, unreserved.amount);
+    }
+
+    // Process slashed accounts
+    for (const slashed of newData.slashed) {
+      ctx.log.info(
+        `Account slashed: ${slashed.address} lost ${slashed.amount} tokens due to misbehavior`
+      );
+      await this.updateAccountBalance(ctx, slashed.address, -slashed.amount);
+    }
+
+    // We don't need to process endowed accounts and dust lost accounts
+    // because they are handled in common transfers and removed accounts
+
     // Process comments
     for (const commentEvt of newData.comments) {
       const sender = await this.getAccountByAddressOrFail(ctx, commentEvt.sender);
@@ -135,26 +190,6 @@ export class DataHandler {
       }
     }
 
-    // Process identities created
-    for (const identity of newData.identitiesCreated) {
-      const account = await this.getAccountByAddressOrFail(ctx, identity.accountId);
-      const newIdentity = new Identity({
-        id: identity.event.id,
-        index: identity.index,
-        // Using the id of the creation event as the name for unconfirmed identities
-        // dirty hack to allow name to be non nullable
-        name: identity.event.id,
-        status: IdentityStatus.Unconfirmed,
-        account,
-        isMember: false,
-        lastChangeOn: identity.blockNumber,
-        createdOn: identity.blockNumber,
-        createdIn: await ctx.store.getOrFail(Event, identity.event.id),
-        expireOn: identity.expireOn,
-      });
-      this.data.identities.set(identity.index, newIdentity);
-    }
-
     // Process identities confirmed
     for (const identity of newData.identitiesConfirmed) {
       const idty = await this.getIdtyByIndexOrFail(ctx, identity.index);
@@ -195,6 +230,7 @@ export class DataHandler {
       ctx.log.info(`Set identity ${index} status to Revoked for reason ${reason.__kind}`);
       idty.status = IdentityStatus.Revoked;
       idty.expireOn = expireOn;
+      idty.firstEligibleUd = 0;
       // only touch if manual revocation
       if (reason.__kind == "User") {
         idty.lastChangeOn = blockNumber;
@@ -221,6 +257,8 @@ export class DataHandler {
       idty.lastChangeOn = idtyChange.blockNumber;
       this.data.identities.set(idtyChange.index, idty);
 
+      ctx.log.info(`Identity ${idtyChange.index} owner key changed to ${idtyChange.accountId}`);
+
       // Related to https://git.duniter.org/nodes/rust/duniter-v2s/-/issues/245.
       // An account can be dissociated from an identity even if the validator is online.
       // To circumvent this problem, we keep a mapping between the account and the last
@@ -238,11 +276,18 @@ export class DataHandler {
     for (const evt of newData.membershipAdded) {
       const { index, expireOn, event } = evt;
 
+      ctx.log.info(`membershipAdded: added}`);
+
+      const currentUd = await this.getLastUniversalDividendOrFail(ctx);
+
+      ctx.log.info(`membershipAdded: Current UD index: ${currentUd.index}`);
+
       const identity = await this.getIdtyByIndexOrFail(ctx, index);
-      identity.status = IdentityStatus.Member
+      identity.status = IdentityStatus.Member;
       identity.expireOn = expireOn;
       identity.isMember = true;
       identity.lastChangeOn = event.block.height;
+      identity.firstEligibleUd = currentUd.index;
 
       // Create membership
       this.data.membershipEvents.push(
@@ -288,6 +333,7 @@ export class DataHandler {
       identity.isMember = false;
       // identity.lastChangeOn = event.block.height; // no: only if active revocation
       identity.expireOn = expireOn
+      identity.firstEligibleUd = 0;
 
       this.data.membershipEvents.push(
         new MembershipEvent({
@@ -540,11 +586,13 @@ export class DataHandler {
 
     // Process universal dividend
     for (const ud of newData.universalDividend) {
-      const { blockNumber, amount, monetaryMass, membersCount, event, timestamp } = ud;
+      const { blockNumber, amount, monetaryMass, membersCount, event, timestamp, index } = ud;
+
       this.data.universalDividend.push(new UniversalDividend({
         id: event.id,
         timestamp: timestamp,
         amount: BigInt(amount),
+        index,
         monetaryMass,
         membersCount: Number(membersCount),
         event: await ctx.store.getOrFail(Event, event.id),
@@ -554,115 +602,96 @@ export class DataHandler {
 
     // Process ud reeval
     for (const udReeval of newData.udReeval) {
-      const { blockNumber, newUdAmount, monetaryMass, membersCount, event, timestamp } =
-        udReeval;
+      const { blockNumber, newUdAmount, monetaryMass, membersCount, event, timestamp } = udReeval;
+
+      const lastUd = await this.getLastUniversalDividendOrFail(ctx);
+
       this.data.udReeval.push(new UdReeval({
         id: event.id,
         timestamp: timestamp,
         newUdAmount: BigInt(newUdAmount),
         monetaryMass,
         membersCount: Number(membersCount),
+        udIndex: lastUd.index,
         event: await ctx.store.getOrFail(Event, event.id),
         blockNumber,
       }));
     }
-  }
-
-  // this is a hack to handle circular dependency between account and identity
-  // account that do not exist in database are created first so that identities can be created after
-  // then accounts can be modified and have linkedIdentity point to the newly created identity
-  async handleNewAccountsApart(ctx: Ctx, newData: NewData) {
-    // Combine account and account link sets to get unique candidates
-    const newAccountCandidates = new Set<Address>([
-      ...newData.accounts.map(a => a.address),
-      ...newData.accountLink.map((link) => link.accountId),
-    ]);
 
-    // Return immediately if no new candidates are present
-    if (newAccountCandidates.size === 0) {
-      return;
-    }
+    // Process ud claims
+    for (const claim of newData.udsClaimed) {
+      // Find identity that owns this account
+      const identityStorage = await ctx.store.findOneOrFail(Identity, {
+        relations: { account: true },
+        where: { account: { id: claim.who } }
+      });
 
-    // Retrieve existing account IDs from the database
-    const existingAccounts = await ctx.store.findBy(Account, {
-      id: In([...newAccountCandidates]),
-    });
-    const existingAccountIds = new Set(
-      existingAccounts.map((account) => account.id)
-    );
+      // Try to get identity from cache, if not found, get it from storage
+      const identity = await this.getIdtyByIndexOrFail(ctx, identityStorage.index);
 
-    // Filter and create accounts that don't already exist
-    const accountsToCreate = newData.accounts
-      .filter(a => !existingAccountIds.has(a.address))
-      .map(a => new Account({ id: a.address, createdOn: a.blockNumber, isActive: true }));
+      // Update firstEligibleUd based on the current UD index
+      const currentUd = await this.getLastUniversalDividendOrFail(ctx);
 
-    if (accountsToCreate.length > 0) {
-      await ctx.store.insert(accountsToCreate);
-      await ctx.store.commit();
-    }
+      ctx.log.info(`udsClaimed: Claiming UD for ${claim.who} at block ${claim.blockNumber} with count ${claim.count} and total ${claim.total}`);
 
-    // Update the Data structure with new accounts
-    for (const account of accountsToCreate) {
-      const existingData = await this.getAccountByAddressOrFail(ctx, account.id);
-      if (existingData) {
-        account.linkedIdentity = existingData.linkedIdentity;
-      }
-      this.data.accounts.set(account.id, account);
+      identity.firstEligibleUd = currentUd.index + 1;
+      this.data.identities.set(identity.index, identity);
     }
   }
 
-  // this is a hack to handle circular dependency between account and identity
-  // identities that are created are handled apart from the ones who only changed
-  async handleNewIdentitiesApart(ctx: Ctx, newData: NewData) {
-    const identities: Array<Identity> = []; // collects identities (only newly created)
-    for (const i of newData.identitiesCreated) {
-      const idty = this.data.identities.get(i.index);
-      assert(idty, "created identities must appear in prepared identities");
-      this.data.identities.delete(i.index); // prevent from trying to add twice
-      identities.push(idty);
-    }
-    if (identities.length === 0) return;
-    // we are sure that all created identities actually do not already exist in database
-    await ctx.store.insert(identities);
-    await ctx.store.commit();
-  }
-
   /// store prepared data into database
   async storeData(ctx: Ctx) {
-    // UPSERT = update or insert if not existing
-    // account can have already existed, been killed, and recreated
-    await ctx.store.upsert([...this.data.accounts.values()]);
-    // identities can have been changed (confirmed, change owner key...) or added (created)
-    await ctx.store.upsert([...this.data.smiths.values()]);
+    // 1. First store identities as many entities depend on them
     await ctx.store.upsert([...this.data.identities.values()]);
-    await ctx.store.upsert([...this.data.validators.values()]);
-    // membership can have been created, renewed, or removed
-    await ctx.store.upsert([...this.data.membershipEvents.values()]);
-    await ctx.store.upsert([...this.data.smithEvents.values()]);
-    // certs can have been created, renewed, or removed
+
+    // 2. Then accounts which may depend on identities
+    for (const account of this.data.accounts.values()) {
+      if (account.linkedIdentity) {
+        const identity = await ctx.store.get(Identity, account.linkedIdentity.id);
+        if (!identity) {
+          account.linkedIdentity = null;
+        }
+      }
+    }
+    await ctx.store.upsert([...this.data.accounts.values()]);
+
+    // 3. Store certifications before their events
     await ctx.store.upsert([...this.data.certification.values()]);
+
+    // 4. Store smiths and their certifications
+    await ctx.store.upsert([...this.data.smiths.values()]);
     await ctx.store.upsert([...this.data.smithCert.values()]);
 
-    // INSERT = these object can not exist before
-    await ctx.store.insert(this.data.comments); // transfers depend on comments
+    // 5. Store validators which depend on smiths
+    await ctx.store.upsert([...this.data.validators.values()]);
+
+    // 6. Store events that depend on previous entities
+    await ctx.store.insert(this.data.comments);
     await ctx.store.insert([...this.data.transfers.values()]);
     await ctx.store.insert(this.data.changeOwnerKey);
-    await ctx.store.insert(this.data.certEvent);
+    await ctx.store.insert(this.data.membershipEvents);
+    await ctx.store.insert(this.data.smithEvents);
+    await ctx.store.insert(this.data.certEvent); // After certification.values()
+
+    // 7. Store independent data
     await ctx.store.insert(this.data.universalDividend);
     await ctx.store.insert(this.data.udReeval);
     await ctx.store.insert(this.data.populationHistories);
 
-    // Apply changes in database
+    // Apply all changes
     await ctx.store.commit();
   }
 
   async getOrCreateAccount(ctx: Ctx, id: Address): Promise<Account> {
-    let account =
-      this.data.accounts.get(id) ?? (await ctx.store.get(Account, id));
+    let account = this.data.accounts.get(id) ?? await ctx.store.get(Account, id);
     if (account == null) {
-      // we need to create it
-      account = new Account({ id, isActive: true });
-      this.data.accounts.set(id, account);
+      account = new Account({
+        id,
+        createdOn: ctx.blocks[0].header.height,
+        isActive: true,
+        balance: 0n,
+        linkedIdentity: null
+      });
     }
     return account;
   }
@@ -674,6 +703,16 @@ export class DataHandler {
     );
   }
 
+  async getLastUniversalDividendOrFail(ctx: Ctx): Promise<UniversalDividend> {
+    if (this.data.universalDividend.length > 0) {
+      return this.data.universalDividend[this.data.universalDividend.length - 1];
+    }
+    return ctx.store.findOneOrFail(UniversalDividend, {
+      where: {},
+      order: { index: 'DESC' }
+    });
+  }
+
   async getIdtyByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Identity> {
     return (
       this.data.identities.get(index) ??
@@ -721,4 +760,10 @@ export class DataHandler {
       });
     }
   }
+
+  async updateAccountBalance(ctx: Ctx, accountId: Address, amount: bigint): Promise<void> {
+    const account = await this.getOrCreateAccount(ctx, accountId);
+    account.balance = (account.balance || 0n) + amount;
+    this.data.accounts.set(accountId, account);
+  }
 }

[Hugo Trentesaux]

Pourquoi est-ce que ça a disparu ?

C'est nécessaire pour afficher l'historique du nombre de membres par exemple :

query MyQuery {
  populationHistory(limit: 1000) {
    activeAccountCount
    blockNumber
    memberCount
    smithCount
  }
}

C'est cassé sur ton instance.

[Hugo Trentesaux]

Not sure you are getting the createdOn right this way. Prefer pass the accountevt.blockNumber.

[Hugo Trentesaux]

Nice way to workaround the hack. Should be ok.

[Hugo Trentesaux]

Would be nice to have a warning in the logs if the account is created here since it is not normal behavior. Silently creating the account outside of new account event can be difficult to debug.

[Hugo Trentesaux]

This hack should not be required in future networks which ensure the account exists before creating identity.

nodes/rust/duniter-v2s#158 (closed)

[Hugo Trentesaux]

Keep the comment you removed. It is non standard behavior and should be explained in the code.

[Hugo Trentesaux]

The reserved amount is still considered as part of the balance (but not free). This implementation choice is valid, but should be explained in detail in the schema.graphql docstrings to avoid confusion (for example when the total balance decreases after distance evaluation request).

[Hugo Trentesaux]

Not sure about dust lost. I have to check that.

[Hugo Trentesaux]

No need to handle that specifically in revoke event. Handling membershipRemoved event should be enough. You can revoke both Member and NotMember identities. It removes membership of Member and gives membershipRemoved event. For NotMember the firstEligibleUd is already 0.

[Hugo Trentesaux]

currentUd is here more like "latest created UD", so the last Universal Dividend created before the membership. But when an identity becomes member, the first UD it's able to claim is the next.

In Duniter it is handled in OnNewMembershipHandler where first_eligible_ud is set to pallet_universal_dividend::init_first_eligible_ud() which is the index of the next UD to be created:

• 1 at the beginning when no UD was created
• 2 after the first UD creation
• 3 after the second UD creation
• ...

[Hugo Trentesaux]

It seems the account relation is not needed anymore because balance changes are handled by their own events, not directly UD claims.

[Hugo Trentesaux]

It seems you are doing it right here, contrary to membership creation. Renaming currentUd to lastUd would make things clearer.

[Hugo Trentesaux]

You already inserted identities before. Here you are just updating their properties.

[Hugo Trentesaux]

You already inserted account before. Here you are just updating their properties. (comment is a bit misleading).

[Hugo Trentesaux]

Does it really happen? It would be very strange since identities are never removed and that they are inserted for sure before the account is linked.

(see comment // should never fail because identity must exist to be able to link itself)

[Hugo Trentesaux]

Nice comments. That's more clear like this. Thanks.

[Hugo Trentesaux]

Is this really the good block? Not the first block of the batch? It should be made homogeneous with other blockNumber in main.ts. So better to pass along the real event block.

[Hugo Trentesaux]

I am not sure how it would work if a batch was large enough to have multiple UDs in it. Maybe the last UD would be selected even if it was created after the block we are currently looking at.

Example:

block   0   1   2   3   4   5   6   7   8   9
ud              1       2       3       4
mship               ↑
batch |       |       |       |       |  
altbt     |       |       |       |       |  

the second batch has:

• block 2 and 3
• membership creation
• ud 1 creation

And getLastUniversalDividendOrFail would return ud 1 which is indeed the last created UD at bock 3.

But the alternative batch (altbt) has:

• block 3 and 4
• membership creation
• ud 2 creation

So getLastUniversalDividendOrFail would return ud 2 which comes after membership creation. In this case, it would give a UD created in the future.

---

[edit]

I'm not sure about what I'm writing. Maybe I'm confused about universalDividend and udReeval which are lists when at most a single element can be in it.

[edit2]

Keep in mind that collectDataFromEvents is run in ctx.blocks.forEach((block) => { loop, and can contain multiple blocks in the context. So what you should be doing is:

1. look into data.universalDividend first and see if there is a UD with block number lower than the one you are looking at
2. look into the database with a where constraint on the blocknumber