Select Git revision
data_handler.ts 26.50 KiB
import { strict as assert } from 'assert';
import { In } from "typeorm";
import {
Account,
Cert,
CertEvent,
ChangeOwnerKey,
Event,
EventType,
Identity,
IdentityStatus,
MembershipEvent,
SmithEvent,
SmithEventType,
Smith,
SmithCert,
SmithStatus,
PopulationHistory,
Transfer,
TxComment,
UdReeval,
UniversalDividend,
Validator,
} from "./model";
import { Address, Ctx, Data, IdtyIndex, NewData } from "./types_custom";
import { hexToString } from "./utils";
import { events } from "./types";
import { getCommentType } from "./comment";
export class DataHandler {
private data: Data;
constructor() {
this.data = {
accounts: new Map(),
identities: new Map(),
smiths: new Map(),
populationHistories: [],
validators: new Map(),
membershipEvents: [],
smithEvents: [],
changeOwnerKey: [],
transfers: new Map(),
certification: new Map(),
certEvent: [],
smithCert: new Map(),
universalDividend: [],
udReeval: [],
comments: []
};
}
async processNewData(newData: NewData, ctx: Ctx) {
// Process population history by adding the value
// of the last point in database.
if (newData.populationHistories) {
const lastHistory = await ctx.store.findOneOrFail(PopulationHistory, {
where: {},
order: { blockNumber: 'DESC' }
});
this.data.populationHistories = newData.populationHistories.map(history => (new PopulationHistory({
activeAccountCount: history.activeAccountCount + lastHistory.activeAccountCount,
memberCount: history.memberCount + lastHistory.memberCount,
smithCount: history.smithCount + lastHistory.smithCount,
blockNumber: history.blockNumber,
id: `population-${history.blockNumber}`,
})));
}
// Process accounts
for (const accountevt of newData.accounts) {
const newAccount = new Account({ id: accountevt.address, createdOn: accountevt.blockNumber, isActive: true });
this.data.accounts.set(accountevt.address, newAccount);
ctx.log.info(`Added account ${accountevt}`);
}
// Process killed accounts
for (const accountId of newData.killedAccounts) {
const killedAccount = await this.getAccountByAddressOrFail(ctx, accountId);
killedAccount.isActive = false;
this.data.accounts.set(accountId, killedAccount);
ctx.log.info(`Killed account ${accountId}`);
}
// Process transfers
for (const transfer of newData.transfers) {
// const fromAccount = await this.getOrCreateAccount(ctx, transfer.from);
// const toAccount = await this.getOrCreateAccount(ctx, transfer.to);
ctx.log.info(
`New transaction: ${transfer.from} to ${transfer.to} of ${transfer.amount} tokens`
);
// should never fail because source of transfer must be an existing account
const fromAccount = await this.getAccountByAddressOrFail(ctx, transfer.from);
// shoud never fail because destination of transfer must be existing account or raise System.NewAccount
const toAccount = await this.getAccountByAddressOrFail(ctx, transfer.to);
const newTransfer = new Transfer({
id: transfer.id,
blockNumber: transfer.blockNumber,
timestamp: transfer.timestamp,
from: fromAccount,
to: toAccount,
amount: transfer.amount,
});
this.data.transfers.set(transfer.id, newTransfer);
}
// Process comments
for (const commentEvt of newData.comments) {
const sender = await this.getAccountByAddressOrFail(ctx, commentEvt.sender);
// create new comment anyway and add it
const remarkHex: string = commentEvt.event.call!.args["remark"] // extract raw data as hex string (temporary before better in Duniter)
const remarkRaw = Buffer.from(remarkHex.substring(2), "hex") // convert it back to bytes
const [commentType, remarkString] = getCommentType(remarkRaw)
const txcomment = new TxComment({
id: commentEvt.event.id,
author: sender,
remarkBytes: remarkRaw,
remark: remarkString,
hash: commentEvt.hash, // raw hex hash (temporary)
type: commentType,
event: await ctx.store.getOrFail(Event, commentEvt.event!.id),
blockNumber: commentEvt.blockNumber,
})
this.data.comments.push(txcomment)
// browse all events in the same extrinsic (batch)
for (const evt of commentEvt.event.extrinsic!.events) {
// only allow commenting transfers
if (evt.name == events.balances.transfer.name) {
const transfer = this.data.transfers.get(evt.id)! // transfers have been handled just before
transfer.comment = txcomment
}
}
}
// Process identities created
for (const identity of newData.identitiesCreated) {
const account = await this.getAccountByAddressOrFail(ctx, identity.accountId); const newIdentity = new Identity({
id: identity.event.id,
index: identity.index,
// Using the id of the creation event as the name for unconfirmed identities
// dirty hack to allow name to be non nullable
name: identity.event.id,
status: IdentityStatus.Unconfirmed,
account,
isMember: false,
lastChangeOn: identity.blockNumber,
createdOn: identity.blockNumber,
createdIn: await ctx.store.getOrFail(Event, identity.event.id),
expireOn: identity.expireOn,
});
this.data.identities.set(identity.index, newIdentity);
}
// Process identities confirmed
for (const identity of newData.identitiesConfirmed) {
const idty = await this.getIdtyByIndexOrFail(ctx, identity.index);
idty.name = hexToString(identity.name);
idty.status = IdentityStatus.Unvalidated;
idty.lastChangeOn = identity.blockNumber;
idty.expireOn = identity.expireOn;
this.data.identities.set(identity.index, idty);
}
// // note: no need for identity validation event since handled in membershipAdded
// // Process identities validated
// for (const identity of newData.identitiesValidated) {
// const idty = await this.getIdtyByIndexOrFail(ctx, identity.index);
// idty.status = IdentityStatus.Member;
// idty.lastChangeOn = identity.blockNumber;
// idty.expireOn = identity.expireOn;
// this.data.identities.set(identity.index, idty);
// }
// Process identities removed
for (const evt of newData.identitiesRemoved) {
const { index, reason } = evt;
const idty = await this.getIdtyWithAccountByIndexOrFail(ctx, index);
ctx.log.info(`Set identity ${index} status to Removed for reason ${reason.__kind}`);
// the only point of keeping removed identities is to block use of their account
// (and maybe pseudo later on)
idty.status = IdentityStatus.Removed;
idty.accountRemoved = idty.account!; // we are sure that a non-removed identity has non-null account
idty.account = null; // having nullable field allows to keep the @unique constraint
this.data.identities.set(index, idty);
}
// Process identities revoked
for (const evt of newData.identitiesRevoked) {
const { index, reason, blockNumber, expireOn } = evt;
const idty = await this.getIdtyByIndexOrFail(ctx, index);
ctx.log.info(`Set identity ${index} status to Revoked for reason ${reason.__kind}`);
idty.status = IdentityStatus.Revoked;
idty.expireOn = expireOn;
// only touch if manual revocation
if (reason.__kind == "User") {
idty.lastChangeOn = blockNumber;
}
this.data.identities.set(index, idty);
}
// Process identity owner key changes
for (const idtyChange of newData.idtyChangedOwnerKey) {
const idty = await this.getIdtyWithAccountByIndexOrFail(ctx, idtyChange.index);
const new_account = await this.getOrCreateAccount(ctx, idtyChange.accountId);
// add change owner key event
this.data.changeOwnerKey.push( new ChangeOwnerKey({
id: idtyChange.id,
identity: idty,
previous: idty.account!, // changing owner key is only possible for a non-removed identity
next: new_account,
blockNumber: idtyChange.blockNumber,
})
);
// after adding the event we can update the identity
idty.account = new_account;
idty.lastChangeOn = idtyChange.blockNumber;
this.data.identities.set(idtyChange.index, idty);
// Related to https://git.duniter.org/nodes/rust/duniter-v2s/-/issues/245.
// An account can be dissociated from an identity even if the validator is online.
// To circumvent this problem, we keep a mapping between the account and the last
// associated identity.
const smith = this.data.smiths.get(idtyChange.index) ??
await ctx.store.findOneBy(Smith, { index: idtyChange.index });
if (smith && smith.smithStatus == SmithStatus.Smith) {
// if an active smith changed owner key, we track this with a new validator
const validator = new Validator({ id: idtyChange.accountId, index: idtyChange.index })
this.data.validators.set(idtyChange.accountId, validator);
}
}
// Process membership added
for (const evt of newData.membershipAdded) {
const { index, expireOn, event } = evt;
const identity = await this.getIdtyByIndexOrFail(ctx, index);
identity.status = IdentityStatus.Member
identity.expireOn = expireOn;
identity.isMember = true;
identity.lastChangeOn = event.block.height;
// Create membership
this.data.membershipEvents.push(
new MembershipEvent({
id: `membership-creation-${index}-${event.id}`,
identity,
eventType: EventType.Creation,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
this.data.identities.set(identity.index, identity);
}
// Process membership renewed
for (const evt of newData.membershipRenewed) {
const { index, expireOn, event } = evt;
const identity = await this.getIdtyByIndexOrFail(ctx, index);
identity.status = IdentityStatus.Member
identity.expireOn = expireOn;
identity.isMember = true;
identity.lastChangeOn = event.block.height;
// Create membership
this.data.membershipEvents.push(
new MembershipEvent({
id: `membership-renewal-${index}-${event.id}`,
identity,
eventType: EventType.Renewal,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
this.data.identities.set(identity.index, identity); }
// Process membership removed
for (const evt of newData.membershipRemoved) {
const { index, event, expireOn } = evt;
const identity = await this.getIdtyByIndexOrFail(ctx, index);
identity.status = IdentityStatus.NotMember;
identity.isMember = false;
// identity.lastChangeOn = event.block.height; // no: only if active revocation
identity.expireOn = expireOn
this.data.membershipEvents.push(
new MembershipEvent({
id: `membership-removal-${index}-${event.id}`,
identity,
eventType: EventType.Removal,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
this.data.identities.set(identity.index, identity);
}
// Process certifications creations
for (const c of newData.certCreation) {
const { issuerId, receiverId, blockNumber, expireOn, event } = c;
// first creation of the cert
let cert = await ctx.store.findOne(Cert, {
relations: { issuer: true, receiver: true },
where: { issuer: { index: issuerId }, receiver: { index: receiverId } },
});
const eventObj = await ctx.store.getOrFail(Event, event.id);
if (cert == null) {
const issuer = await this.getIdtyByIndexOrFail(ctx, issuerId);
const receiver = await this.getIdtyByIndexOrFail(ctx, receiverId);
cert = new Cert({
id: event.id,
isActive: true,
issuer,
receiver,
createdOn: blockNumber,
createdIn: eventObj,
updatedOn: blockNumber,
updatedIn: eventObj,
expireOn
});
// the cert has already existed, expired, and is created again
// we update it accordingly
} else {
cert.isActive = true;
cert.updatedOn = blockNumber;
cert.updatedIn = eventObj;
cert.expireOn = expireOn;
}
// update cert and add event
this.data.certification.set([issuerId, receiverId], cert);
this.data.certEvent.push(
new CertEvent({
id: event.id,
cert,
eventType: EventType.Creation,
event: eventObj,
blockNumber,
})
);
}
// Process certifications renewals for (const c of newData.certRenewal) {
const { issuerId, receiverId, blockNumber, expireOn, event } = c;
// should never fail because cert renewal can only happen on existing cert
// and can not be renewed at the same block as created (delay)
const cert = await ctx.store.findOneOrFail(Cert, {
relations: { issuer: true, receiver: true },
where: { issuer: { index: issuerId }, receiver: { index: receiverId } },
});
const eventObj = await ctx.store.getOrFail(Event, event.id);
// update expiration date
cert.expireOn = expireOn;
cert.updatedOn = blockNumber;
cert.updatedIn = eventObj;
this.data.certification.set([issuerId, receiverId], cert);
this.data.certEvent.push(
new CertEvent({
id: event.id,
cert,
eventType: EventType.Renewal,
event: eventObj,
blockNumber,
})
);
}
// Process validators
for (const { block, validatorId } of newData.validators) {
// validator must already exist because it was created at the same time as smith, which is necessary to compute block
const validator = await this.getValidatorByAddressOrFail(ctx, validatorId);
// we are sure that a smith exist because they are created at the same time
const smith = await this.getSmithByIndexOrFail(ctx, validator.index);
smith.forged += 1;
smith.lastForged = block
this.data.smiths.set(smith.index, smith);
}
// Process certifications removals
for (const c of newData.certRemoval) {
const { issuerId, receiverId, blockNumber, event } = c;
// should never fail because cert removal can only happen on existing cert
// and cert should not be removed at their creation block
const cert = await ctx.store.findOneOrFail(Cert, {
relations: { issuer: true, receiver: true },
where: { issuer: { index: issuerId }, receiver: { index: receiverId } },
});
// update cert
cert.isActive = false;
this.data.certification.set([issuerId, receiverId], cert);
this.data.certEvent.push(
new CertEvent({
id: event.id,
cert,
eventType: EventType.Removal,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber,
})
);
}
// Process Smith certifications
for (const c of newData.smithCertAdded) {
const { id, issuerId, receiverId, createdOn } = c;
let cert = await ctx.store.findOne(SmithCert, {
relations: { issuer: true, receiver: true },
where: { issuer: { index: issuerId }, receiver: { index: receiverId } },
});
if (cert == null) {
const issuer = await this.getSmithByIndexOrFail(ctx, issuerId);
const receiver = await this.getSmithByIndexOrFail(ctx, receiverId);
cert = new SmithCert({ id,
issuer,
receiver,
createdOn,
});
} else {
cert.createdOn = createdOn;
}
this.data.smithCert.set([issuerId, receiverId], cert);
}
// Process remove smith cert
for (const smithCertRemoved of newData.smithCertRemoved) {
const { issuerId, receiverId } = smithCertRemoved;
const smithCert = await ctx.store.findOneOrFail(SmithCert, {
where: { issuer: { index: issuerId }, receiver: { index: receiverId } },
});
await ctx.store.remove(smithCert);
}
// Process Smith invitation sent
for (const invitedSmith of newData.smithInvited) {
const { idtyIndex, event } = invitedSmith;
const smith = await this.getOrCreateSmith(ctx, idtyIndex);
smith.smithStatus = SmithStatus.Invited;
smith.lastChanged = event.block.height;
this.data.smiths.set(idtyIndex, smith);
this.data.smithEvents.push(
new SmithEvent({
id: `smith-invited-${idtyIndex}-${event.id}`,
smith,
eventType: SmithEventType.Invited,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
}
// Process Smith invitation accepted
for (const acceptedSmithInvitations of newData.smithAccepted) {
const { idtyIndex, event } = acceptedSmithInvitations;
const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex);
smith.smithStatus = SmithStatus.Pending;
smith.lastChanged = event.block.height;
this.data.smiths.set(idtyIndex, smith);
this.data.smithEvents.push(
new SmithEvent({
id: `smith-accepted-${idtyIndex}-${event.id}`,
smith,
eventType: SmithEventType.Accepted,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
}
// Process Smith promotion
for (const promotedSmith of newData.smithPromoted) {
const { idtyIndex, event } = promotedSmith;
const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex);
smith.smithStatus = SmithStatus.Smith;
smith.lastChanged = event.block.height;
this.data.smiths.set(idtyIndex, smith);
this.data.smithEvents.push(
new SmithEvent({
id: `smith-promoted-${idtyIndex}-${event.id}`, smith,
eventType: SmithEventType.Promoted,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
const identity = await this.getIdtyWithAccountByIndexOrFail(ctx, idtyIndex);
const smithAccount = identity.account!; // a smith can only be a non-removed identity
const validator = new Validator({ id: smithAccount.id, index: idtyIndex })
this.data.validators.set(smithAccount.id, validator);
}
// Process Smith exlusion
for (const excludedSmith of newData.smithExcluded) {
const { idtyIndex, event } = excludedSmith;
const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex);
smith.smithStatus = SmithStatus.Excluded;
smith.lastChanged = event.block.height;
this.data.smiths.set(idtyIndex, smith);
this.data.smithEvents.push(
new SmithEvent({
id: `smith-excluded-${idtyIndex}-${event.id}`,
smith,
eventType: SmithEventType.Excluded,
event: await ctx.store.getOrFail(Event, event.id),
blockNumber: event.block.height,
})
);
}
// Process account links
for (const link of newData.accountLink) {
// we can link an identity to a non-existing account
const account = await this.getOrCreateAccount(ctx, link.accountId);
// should never fail because identity must exist to be able to link itself
const idty = await this.getIdtyByIndexOrFail(ctx, link.index);
account.linkedIdentity = idty;
this.data.accounts.set(account.id, account);
}
// Process account unlinks
for (const unlink of newData.accountUnlink) {
// should never fail because account must exist to unlink due to tx fees
const account = await this.getAccountByAddressOrFail(ctx, unlink.accountId);
account.linkedIdentity = null;
this.data.accounts.set(account.id, account);
}
// Process universal dividend
for (const ud of newData.universalDividend) {
const { blockNumber, amount, monetaryMass, membersCount, event, timestamp } = ud;
this.data.universalDividend.push(new UniversalDividend({
id: event.id,
timestamp: timestamp,
amount: BigInt(amount),
monetaryMass,
membersCount: Number(membersCount),
event: await ctx.store.getOrFail(Event, event.id),
blockNumber,
}));
}
// Process ud reeval
for (const udReeval of newData.udReeval) {
const { blockNumber, newUdAmount, monetaryMass, membersCount, event, timestamp } =
udReeval;
this.data.udReeval.push(new UdReeval({
id: event.id, timestamp: timestamp,
newUdAmount: BigInt(newUdAmount),
monetaryMass,
membersCount: Number(membersCount),
event: await ctx.store.getOrFail(Event, event.id),
blockNumber,
}));
}
}
// this is a hack to handle circular dependency between account and identity
// account that do not exist in database are created first so that identities can be created after
// then accounts can be modified and have linkedIdentity point to the newly created identity
async handleNewAccountsApart(ctx: Ctx, newData: NewData) {
// Combine account and account link sets to get unique candidates
const newAccountCandidates = new Set<Address>([
...newData.accounts.map(a => a.address),
...newData.accountLink.map((link) => link.accountId),
]);
// Return immediately if no new candidates are present
if (newAccountCandidates.size === 0) {
return;
}
// Retrieve existing account IDs from the database
const existingAccounts = await ctx.store.findBy(Account, {
id: In([...newAccountCandidates]),
});
const existingAccountIds = new Set(
existingAccounts.map((account) => account.id)
);
// Filter and create accounts that don't already exist
const accountsToCreate = newData.accounts
.filter(a => !existingAccountIds.has(a.address))
.map(a => new Account({ id: a.address, createdOn: a.blockNumber, isActive: true }));
if (accountsToCreate.length > 0) {
await ctx.store.insert(accountsToCreate);
await ctx.store.commit();
}
// Update the Data structure with new accounts
for (const account of accountsToCreate) {
const existingData = await this.getAccountByAddressOrFail(ctx, account.id);
if (existingData) {
account.linkedIdentity = existingData.linkedIdentity;
}
this.data.accounts.set(account.id, account);
}
}
// this is a hack to handle circular dependency between account and identity
// identities that are created are handled apart from the ones who only changed
async handleNewIdentitiesApart(ctx: Ctx, newData: NewData) {
const identities: Array<Identity> = []; // collects identities (only newly created)
for (const i of newData.identitiesCreated) {
const idty = this.data.identities.get(i.index);
assert(idty, "created identities must appear in prepared identities");
this.data.identities.delete(i.index); // prevent from trying to add twice
identities.push(idty);
}
if (identities.length === 0) return;
// we are sure that all created identities actually do not already exist in database
await ctx.store.insert(identities);
await ctx.store.commit();
}
/// store prepared data into database async storeData(ctx: Ctx) {
// UPSERT = update or insert if not existing
// account can have already existed, been killed, and recreated
await ctx.store.upsert([...this.data.accounts.values()]);
// identities can have been changed (confirmed, change owner key...) or added (created)
await ctx.store.upsert([...this.data.smiths.values()]);
await ctx.store.upsert([...this.data.identities.values()]);
await ctx.store.upsert([...this.data.validators.values()]);
// membership can have been created, renewed, or removed
await ctx.store.upsert([...this.data.membershipEvents.values()]);
await ctx.store.upsert([...this.data.smithEvents.values()]);
// certs can have been created, renewed, or removed
await ctx.store.upsert([...this.data.certification.values()]);
await ctx.store.upsert([...this.data.smithCert.values()]);
// INSERT = these object can not exist before
await ctx.store.insert(this.data.comments); // transfers depend on comments
await ctx.store.insert([...this.data.transfers.values()]);
await ctx.store.insert(this.data.changeOwnerKey);
await ctx.store.insert(this.data.certEvent);
await ctx.store.insert(this.data.universalDividend);
await ctx.store.insert(this.data.udReeval);
await ctx.store.insert(this.data.populationHistories);
// Apply changes in database
await ctx.store.commit();
}
async getOrCreateAccount(ctx: Ctx, id: Address): Promise<Account> {
let account =
this.data.accounts.get(id) ?? (await ctx.store.get(Account, id));
if (account == null) {
// we need to create it
account = new Account({ id, isActive: true });
this.data.accounts.set(id, account);
}
return account;
}
async getAccountByAddressOrFail(ctx: Ctx, address: Address): Promise<Account> {
return (
this.data.accounts.get(address) ??
ctx.store.findOneByOrFail(Account, { id: address })
);
}
async getIdtyByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Identity> {
return (
this.data.identities.get(index) ??
ctx.store.findOneByOrFail(Identity, { index })
);
}
async getSmithByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Smith> {
return (
this.data.smiths.get(index) ??
ctx.store.findOneByOrFail(Smith, { index })
);
}
async getIdtyWithAccountByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Identity> {
return (
this.data.identities.get(index) ??
ctx.store.findOneOrFail(Identity, {
relations: { account: true },
where: { index: index },
}))
}
async getValidatorByAddressOrFail(ctx: Ctx, address: Address): Promise<Validator> { return (
this.data.validators.get(address) ??
ctx.store.findOneByOrFail(Validator, { id: address })
);
}
async getOrCreateSmith(ctx: Ctx, index: IdtyIndex): Promise<Smith> {
const smith =
this.data.smiths.get(index) ??
await ctx.store.findOneBy(Smith, { index });
if (smith) {
return smith;
}
else {
const identity = await this.getIdtyByIndexOrFail(ctx, index);
return new Smith({
id: `smith_${index}`,
index: index,
identity: identity,
forged: 0,
});
}
}
}