diff --git a/Dockerfile b/Dockerfile index c23de2c9f710ddad5bc23f3ab1d5047480317f54..53eb94831aa9a9a9cbcb5a1beea2a668030c699a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN pnpm build # RUN ./scripts/download_genesis.sh # --- use this if you want to copy from pre-downloaded files -FROM node as input +FROM node AS input WORKDIR /squid COPY /input/ input/ @@ -51,10 +51,10 @@ RUN npm i -g @subsquid/commands && mv $(which squid-commands) /usr/local/bin/sqd # environment variables # ENV PROCESSOR_PROMETHEUS_PORT 3000 -ENV GENESIS_FILE "./input/gdev.json" -ENV HIST_GEN_FILE "./input/genesis.json" -ENV HIST_BLOCK_FILE "./input/block_hist.json" -ENV HIST_TX_FILE "./input/tx_hist.json" -ENV HIST_CERT_FILE "./input/cert_hist.json" +ENV GENESIS_FILE="./input/gdev.json" +ENV HIST_GEN_FILE="./input/genesis.json" +ENV HIST_BLOCK_FILE="./input/block_hist.json" +ENV HIST_TX_FILE="./input/tx_hist.json" +ENV HIST_CERT_FILE="./input/cert_hist.json" # ENTRYPOINT ["pnpm", "exec", "squid-commands", "process:prod"] \ No newline at end of file diff --git a/assets/sql/EnumsMigration_down.sql b/assets/sql/EnumsMigration_down.sql index 62dc4ce0fbb924662fd4f60d51d4cdf229dd5c0b..0b47d8029baae8d2a7b88782d7d5bcf5c1c18c49 100644 --- a/assets/sql/EnumsMigration_down.sql +++ b/assets/sql/EnumsMigration_down.sql @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS item_type; DROP TABLE IF EXISTS counter_level; -DROP TABLE IF EXISTS identity_status; DROP TABLE IF EXISTS smith_status; +DROP TABLE IF EXISTS identity_status; DROP TABLE IF EXISTS event_type; +DROP TABLE IF EXISTS smith_event_type; DROP TABLE IF EXISTS comment_type; diff --git a/assets/sql/EnumsMigration_up.sql b/assets/sql/EnumsMigration_up.sql index e8ffed2e57fed36936e79f6872f5f755f55eba88..a4f54644b29209ce162e6632a01d1a62190dca56 100644 --- a/assets/sql/EnumsMigration_up.sql +++ b/assets/sql/EnumsMigration_up.sql @@ -10,6 +10,13 @@ INSERT INTO counter_level (value) VALUES ('Pallet'); INSERT INTO counter_level (value) VALUES ('Item'); ALTER TABLE items_counter DROP COLUMN IF EXISTS level; ALTER TABLE items_counter ADD COLUMN level TEXT REFERENCES counter_level(value); +CREATE TABLE smith_status (value TEXT PRIMARY KEY); +INSERT INTO smith_status (value) VALUES ('Invited'); +INSERT INTO smith_status (value) VALUES ('Pending'); +INSERT INTO smith_status (value) VALUES ('Smith'); +INSERT INTO smith_status (value) VALUES ('Excluded'); +ALTER TABLE smith DROP COLUMN IF EXISTS smith_status; +ALTER TABLE smith ADD COLUMN smith_status TEXT REFERENCES smith_status(value); CREATE TABLE identity_status (value TEXT PRIMARY KEY); INSERT INTO identity_status (value) VALUES ('Unconfirmed'); INSERT INTO identity_status (value) VALUES ('Unvalidated'); @@ -19,13 +26,6 @@ INSERT INTO identity_status (value) VALUES ('Revoked'); INSERT INTO identity_status (value) VALUES ('Removed'); ALTER TABLE identity DROP COLUMN IF EXISTS status; ALTER TABLE identity ADD COLUMN status TEXT REFERENCES identity_status(value); -CREATE TABLE smith_status (value TEXT PRIMARY KEY); -INSERT INTO smith_status (value) VALUES ('Invited'); -INSERT INTO smith_status (value) VALUES ('Pending'); -INSERT INTO smith_status (value) VALUES ('Smith'); -INSERT INTO smith_status (value) VALUES ('Excluded'); -ALTER TABLE identity DROP COLUMN IF EXISTS smith_status; -ALTER TABLE identity ADD COLUMN smith_status TEXT REFERENCES smith_status(value); CREATE TABLE event_type (value TEXT PRIMARY KEY); INSERT INTO event_type (value) VALUES ('Creation'); INSERT INTO event_type (value) VALUES ('Renewal'); @@ -34,6 +34,13 @@ ALTER TABLE cert_event DROP COLUMN IF EXISTS event_type; ALTER TABLE cert_event ADD COLUMN event_type TEXT REFERENCES event_type(value); ALTER TABLE membership_event DROP COLUMN IF EXISTS event_type; ALTER TABLE membership_event ADD COLUMN event_type TEXT REFERENCES event_type(value); +CREATE TABLE smith_event_type (value TEXT PRIMARY KEY); +INSERT INTO smith_event_type (value) VALUES ('Invited'); +INSERT INTO smith_event_type (value) VALUES ('Accepted'); +INSERT INTO smith_event_type (value) VALUES ('Promoted'); +INSERT INTO smith_event_type (value) VALUES ('Excluded'); +ALTER TABLE smith_event DROP COLUMN IF EXISTS event_type; +ALTER TABLE smith_event ADD COLUMN event_type TEXT REFERENCES smith_event_type(value); CREATE TABLE comment_type (value TEXT PRIMARY KEY); INSERT INTO comment_type (value) VALUES ('Cid'); INSERT INTO comment_type (value) VALUES ('Ascii'); diff --git a/db/migrations/1717671693690-Data.js b/db/migrations/1725878455111-Data.js similarity index 84% rename from db/migrations/1717671693690-Data.js rename to db/migrations/1725878455111-Data.js index b2b750cdca32fa48fe516212fbe2e7a994de7a3a..1d6f5a989626af94448f316640e74d2d19964b87 100644 --- a/db/migrations/1717671693690-Data.js +++ b/db/migrations/1725878455111-Data.js @@ -1,5 +1,5 @@ -module.exports = class Data1717671693690 { - name = 'Data1717671693690' +module.exports = class Data1725878455111 { + name = 'Data1725878455111' async up(db) { await db.query(`CREATE TABLE "event" ("id" character varying NOT NULL, "index" integer NOT NULL, "phase" text NOT NULL, "pallet" text NOT NULL, "name" text NOT NULL, "args" jsonb, "args_str" text array, "block_id" character varying, "extrinsic_id" character varying, "call_id" character varying, CONSTRAINT "PK_30c2f3bbaf6d34a55f8ae6e4614" PRIMARY KEY ("id"))`) @@ -41,9 +41,6 @@ module.exports = class Data1717671693690 { await db.query(`CREATE INDEX "IDX_262e29ab91c8ebc727cc518f2f" ON "cert" ("receiver_id") `) await db.query(`CREATE INDEX "IDX_ad35ca166ad24ecea43d7ebfca" ON "cert" ("created_in_id") `) await db.query(`CREATE INDEX "IDX_5fbefe3a497e898aff45e44f50" ON "cert" ("updated_in_id") `) - await db.query(`CREATE TABLE "smith_cert" ("id" character varying NOT NULL, "created_on" integer NOT NULL, "issuer_id" character varying, "receiver_id" character varying, CONSTRAINT "PK_ae2ef36c9f6d40348c86230fd35" PRIMARY KEY ("id"))`) - await db.query(`CREATE INDEX "IDX_ae67cbd087fcea0e1ec2f70cd0" ON "smith_cert" ("issuer_id") `) - await db.query(`CREATE INDEX "IDX_5e414c1d12af16165881a16b63" ON "smith_cert" ("receiver_id") `) await db.query(`CREATE TABLE "membership_event" ("id" character varying NOT NULL, "event_type" character varying(8) NOT NULL, "block_number" integer NOT NULL, "identity_id" character varying, "event_id" character varying, CONSTRAINT "PK_a5bbbf1d51331d1ff4e6419c48e" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_fbbd75d84ab6cc2aafeaf37a03" ON "membership_event" ("identity_id") `) await db.query(`CREATE INDEX "IDX_cebe39c6f75881fb55dbe13741" ON "membership_event" ("event_id") `) @@ -54,9 +51,10 @@ module.exports = class Data1717671693690 { await db.query(`CREATE INDEX "IDX_9a9f8cdcae54d8b4375d70fe8b" ON "change_owner_key" ("next_id") `) await db.query(`CREATE TABLE "ud_history" ("id" character varying NOT NULL, "amount" integer NOT NULL, "block_number" integer NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "identity_id" character varying, CONSTRAINT "PK_29dc558594261d7b2845876b6e1" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_a481d5b96ba0e6d8237dc733a3" ON "ud_history" ("identity_id") `) - await db.query(`CREATE TABLE "identity" ("id" character varying NOT NULL, "index" integer NOT NULL, "name" text NOT NULL, "status" character varying(11) NOT NULL, "created_on" integer NOT NULL, "last_change_on" integer NOT NULL, "smith_status" character varying(8), "is_member" boolean NOT NULL, "expire_on" integer NOT NULL, "account_id" character varying, "created_in_id" character varying, CONSTRAINT "REL_bafa9e6c71c3f69cef6602a809" UNIQUE ("account_id"), CONSTRAINT "PK_ff16a44186b286d5e626178f726" PRIMARY KEY ("id"))`) + await db.query(`CREATE TABLE "identity" ("id" character varying NOT NULL, "index" integer NOT NULL, "name" text NOT NULL, "status" character varying(11) NOT NULL, "created_on" integer NOT NULL, "last_change_on" integer NOT NULL, "is_member" boolean NOT NULL, "expire_on" integer NOT NULL, "account_id" character varying, "account_removed_id" character varying, "created_in_id" character varying, CONSTRAINT "REL_bafa9e6c71c3f69cef6602a809" UNIQUE ("account_id"), CONSTRAINT "PK_ff16a44186b286d5e626178f726" PRIMARY KEY ("id"))`) await db.query(`CREATE UNIQUE INDEX "IDX_6f883c7979ea8dff46327f67cc" ON "identity" ("index") `) await db.query(`CREATE UNIQUE INDEX "IDX_bafa9e6c71c3f69cef6602a809" ON "identity" ("account_id") `) + await db.query(`CREATE INDEX "IDX_29e3a9afec095a892e1a3bddd3" ON "identity" ("account_removed_id") `) await db.query(`CREATE INDEX "IDX_883ba5be237fba47f2a2f39145" ON "identity" ("name") `) await db.query(`CREATE INDEX "IDX_ee232f862b258f533e70bbb24d" ON "identity" ("status") `) await db.query(`CREATE INDEX "IDX_666fdfaf3a12c96e95ab0a0b31" ON "identity" ("created_in_id") `) @@ -71,12 +69,26 @@ module.exports = class Data1717671693690 { await db.query(`CREATE INDEX "IDX_f4007436c1b546ede08a4fd7ab" ON "transfer" ("amount") `) await db.query(`CREATE INDEX "IDX_2a4e1dce9f72514cd28f554ee2" ON "transfer" ("event_id") `) await db.query(`CREATE INDEX "IDX_76cf30dd4464ed95ad44c5bb61" ON "transfer" ("comment_id") `) - await db.query(`CREATE TABLE "account" ("id" character varying NOT NULL, "linked_identity_id" character varying, CONSTRAINT "PK_54115ee388cdb6d86bb4bf5b2ea" PRIMARY KEY ("id"))`) + await db.query(`CREATE TABLE "account" ("id" character varying NOT NULL, "is_active" boolean NOT NULL, "linked_identity_id" character varying, CONSTRAINT "PK_54115ee388cdb6d86bb4bf5b2ea" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_73d14d249a6dcf9abe42eaa657" ON "account" ("linked_identity_id") `) + await db.query(`CREATE TABLE "validator" ("id" character varying NOT NULL, "index" integer NOT NULL, CONSTRAINT "PK_ae0a943022c24bd60e7161e0fad" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_49842e345355a71ff691c7a1ee" ON "validator" ("index") `) + await db.query(`CREATE TABLE "smith_cert" ("id" character varying NOT NULL, "created_on" integer NOT NULL, "issuer_id" character varying, "receiver_id" character varying, CONSTRAINT "PK_ae2ef36c9f6d40348c86230fd35" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_ae67cbd087fcea0e1ec2f70cd0" ON "smith_cert" ("issuer_id") `) + await db.query(`CREATE INDEX "IDX_5e414c1d12af16165881a16b63" ON "smith_cert" ("receiver_id") `) + await db.query(`CREATE TABLE "smith_event" ("id" character varying NOT NULL, "event_type" character varying(8) NOT NULL, "block_number" integer NOT NULL, "smith_id" character varying, "event_id" character varying, CONSTRAINT "PK_3904958b2376145f2378496d7a2" PRIMARY KEY ("id"))`) + await db.query(`CREATE INDEX "IDX_8375d29344115baadd964eca6a" ON "smith_event" ("smith_id") `) + await db.query(`CREATE INDEX "IDX_52efc3145aa64c6cef0dfc05a7" ON "smith_event" ("event_id") `) + await db.query(`CREATE INDEX "IDX_b5906661a91fda1c36d14ede86" ON "smith_event" ("block_number") `) + await db.query(`CREATE TABLE "smith" ("id" character varying NOT NULL, "index" integer NOT NULL, "smith_status" character varying(8), "last_changed" integer, "forged" integer NOT NULL, "last_forged" integer, "validators_id" text array, "identity_id" character varying, CONSTRAINT "REL_b7aab883baf637058d84fb3033" UNIQUE ("identity_id"), CONSTRAINT "PK_4e20a10e9810ca441a39ebb3f8e" PRIMARY KEY ("id"))`) + await db.query(`CREATE UNIQUE INDEX "IDX_2f13e9e1f0106f4dca107fdd1f" ON "smith" ("index") `) + await db.query(`CREATE UNIQUE INDEX "IDX_b7aab883baf637058d84fb3033" ON "smith" ("identity_id") `) await db.query(`CREATE TABLE "universal_dividend" ("id" character varying NOT NULL, "block_number" integer NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "amount" integer NOT NULL, "monetary_mass" numeric NOT NULL, "members_count" integer NOT NULL, "event_id" character varying, CONSTRAINT "PK_f7557418097ca486e193ded801c" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_ad1b348c17129da2cc28fceb83" ON "universal_dividend" ("event_id") `) await db.query(`CREATE TABLE "ud_reeval" ("id" character varying NOT NULL, "block_number" integer NOT NULL, "timestamp" TIMESTAMP WITH TIME ZONE NOT NULL, "new_ud_amount" integer NOT NULL, "monetary_mass" numeric NOT NULL, "members_count" integer NOT NULL, "event_id" character varying, CONSTRAINT "PK_1d29de2d3c63d6169a73d7ec12b" PRIMARY KEY ("id"))`) await db.query(`CREATE INDEX "IDX_38a22f60573ad32c42ca0c2c11" ON "ud_reeval" ("event_id") `) + await db.query(`CREATE TABLE "population_history" ("id" character varying NOT NULL, "smith_count" integer NOT NULL, "member_count" integer NOT NULL, "active_account_count" integer NOT NULL, "block_number" integer NOT NULL, CONSTRAINT "PK_8561d4384a9c5c911df04b60bac" PRIMARY KEY ("id"))`) + await db.query(`CREATE UNIQUE INDEX "IDX_27f5d94ffa8d638a40a0b564a4" ON "population_history" ("block_number") `) await db.query(`ALTER TABLE "event" ADD CONSTRAINT "FK_2b0d35d675c4f99751855c45021" FOREIGN KEY ("block_id") REFERENCES "block"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "event" ADD CONSTRAINT "FK_129efedcb305c80256db2d57a59" FOREIGN KEY ("extrinsic_id") REFERENCES "extrinsic"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "event" ADD CONSTRAINT "FK_83cf1bd59aa4521ed882fa51452" FOREIGN KEY ("call_id") REFERENCES "call"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) @@ -91,8 +103,6 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "cert" ADD CONSTRAINT "FK_262e29ab91c8ebc727cc518f2fb" FOREIGN KEY ("receiver_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "cert" ADD CONSTRAINT "FK_ad35ca166ad24ecea43d7ebfca9" FOREIGN KEY ("created_in_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "cert" ADD CONSTRAINT "FK_5fbefe3a497e898aff45e44f504" FOREIGN KEY ("updated_in_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) - await db.query(`ALTER TABLE "smith_cert" ADD CONSTRAINT "FK_ae67cbd087fcea0e1ec2f70cd04" FOREIGN KEY ("issuer_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) - await db.query(`ALTER TABLE "smith_cert" ADD CONSTRAINT "FK_5e414c1d12af16165881a16b638" FOREIGN KEY ("receiver_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "membership_event" ADD CONSTRAINT "FK_fbbd75d84ab6cc2aafeaf37a03f" FOREIGN KEY ("identity_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "membership_event" ADD CONSTRAINT "FK_cebe39c6f75881fb55dbe137418" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "change_owner_key" ADD CONSTRAINT "FK_af577baa612d86d98a1ae583438" FOREIGN KEY ("identity_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) @@ -100,6 +110,7 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "change_owner_key" ADD CONSTRAINT "FK_9a9f8cdcae54d8b4375d70fe8be" FOREIGN KEY ("next_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "ud_history" ADD CONSTRAINT "FK_a481d5b96ba0e6d8237dc733a31" FOREIGN KEY ("identity_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "identity" ADD CONSTRAINT "FK_bafa9e6c71c3f69cef6602a8095" FOREIGN KEY ("account_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "identity" ADD CONSTRAINT "FK_29e3a9afec095a892e1a3bddd39" FOREIGN KEY ("account_removed_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "identity" ADD CONSTRAINT "FK_666fdfaf3a12c96e95ab0a0b31c" FOREIGN KEY ("created_in_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "tx_comment" ADD CONSTRAINT "FK_39157ef296298a13cb591f8abab" FOREIGN KEY ("author_id") REFERENCES "account"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "tx_comment" ADD CONSTRAINT "FK_7c33e1189dfd717140393b2e30d" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) @@ -108,6 +119,11 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "transfer" ADD CONSTRAINT "FK_2a4e1dce9f72514cd28f554ee2d" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "transfer" ADD CONSTRAINT "FK_76cf30dd4464ed95ad44c5bb616" FOREIGN KEY ("comment_id") REFERENCES "tx_comment"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "account" ADD CONSTRAINT "FK_73d14d249a6dcf9abe42eaa6573" FOREIGN KEY ("linked_identity_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "smith_cert" ADD CONSTRAINT "FK_ae67cbd087fcea0e1ec2f70cd04" FOREIGN KEY ("issuer_id") REFERENCES "smith"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "smith_cert" ADD CONSTRAINT "FK_5e414c1d12af16165881a16b638" FOREIGN KEY ("receiver_id") REFERENCES "smith"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "smith_event" ADD CONSTRAINT "FK_8375d29344115baadd964eca6ab" FOREIGN KEY ("smith_id") REFERENCES "smith"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "smith_event" ADD CONSTRAINT "FK_52efc3145aa64c6cef0dfc05a7c" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) + await db.query(`ALTER TABLE "smith" ADD CONSTRAINT "FK_b7aab883baf637058d84fb30335" FOREIGN KEY ("identity_id") REFERENCES "identity"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "universal_dividend" ADD CONSTRAINT "FK_ad1b348c17129da2cc28fceb830" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) await db.query(`ALTER TABLE "ud_reeval" ADD CONSTRAINT "FK_38a22f60573ad32c42ca0c2c111" FOREIGN KEY ("event_id") REFERENCES "event"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`) } @@ -152,9 +168,6 @@ module.exports = class Data1717671693690 { await db.query(`DROP INDEX "public"."IDX_262e29ab91c8ebc727cc518f2f"`) await db.query(`DROP INDEX "public"."IDX_ad35ca166ad24ecea43d7ebfca"`) await db.query(`DROP INDEX "public"."IDX_5fbefe3a497e898aff45e44f50"`) - await db.query(`DROP TABLE "smith_cert"`) - await db.query(`DROP INDEX "public"."IDX_ae67cbd087fcea0e1ec2f70cd0"`) - await db.query(`DROP INDEX "public"."IDX_5e414c1d12af16165881a16b63"`) await db.query(`DROP TABLE "membership_event"`) await db.query(`DROP INDEX "public"."IDX_fbbd75d84ab6cc2aafeaf37a03"`) await db.query(`DROP INDEX "public"."IDX_cebe39c6f75881fb55dbe13741"`) @@ -168,6 +181,7 @@ module.exports = class Data1717671693690 { await db.query(`DROP TABLE "identity"`) await db.query(`DROP INDEX "public"."IDX_6f883c7979ea8dff46327f67cc"`) await db.query(`DROP INDEX "public"."IDX_bafa9e6c71c3f69cef6602a809"`) + await db.query(`DROP INDEX "public"."IDX_29e3a9afec095a892e1a3bddd3"`) await db.query(`DROP INDEX "public"."IDX_883ba5be237fba47f2a2f39145"`) await db.query(`DROP INDEX "public"."IDX_ee232f862b258f533e70bbb24d"`) await db.query(`DROP INDEX "public"."IDX_666fdfaf3a12c96e95ab0a0b31"`) @@ -184,10 +198,24 @@ module.exports = class Data1717671693690 { await db.query(`DROP INDEX "public"."IDX_76cf30dd4464ed95ad44c5bb61"`) await db.query(`DROP TABLE "account"`) await db.query(`DROP INDEX "public"."IDX_73d14d249a6dcf9abe42eaa657"`) + await db.query(`DROP TABLE "validator"`) + await db.query(`DROP INDEX "public"."IDX_49842e345355a71ff691c7a1ee"`) + await db.query(`DROP TABLE "smith_cert"`) + await db.query(`DROP INDEX "public"."IDX_ae67cbd087fcea0e1ec2f70cd0"`) + await db.query(`DROP INDEX "public"."IDX_5e414c1d12af16165881a16b63"`) + await db.query(`DROP TABLE "smith_event"`) + await db.query(`DROP INDEX "public"."IDX_8375d29344115baadd964eca6a"`) + await db.query(`DROP INDEX "public"."IDX_52efc3145aa64c6cef0dfc05a7"`) + await db.query(`DROP INDEX "public"."IDX_b5906661a91fda1c36d14ede86"`) + await db.query(`DROP TABLE "smith"`) + await db.query(`DROP INDEX "public"."IDX_2f13e9e1f0106f4dca107fdd1f"`) + await db.query(`DROP INDEX "public"."IDX_b7aab883baf637058d84fb3033"`) await db.query(`DROP TABLE "universal_dividend"`) await db.query(`DROP INDEX "public"."IDX_ad1b348c17129da2cc28fceb83"`) await db.query(`DROP TABLE "ud_reeval"`) await db.query(`DROP INDEX "public"."IDX_38a22f60573ad32c42ca0c2c11"`) + await db.query(`DROP TABLE "population_history"`) + await db.query(`DROP INDEX "public"."IDX_27f5d94ffa8d638a40a0b564a4"`) await db.query(`ALTER TABLE "event" DROP CONSTRAINT "FK_2b0d35d675c4f99751855c45021"`) await db.query(`ALTER TABLE "event" DROP CONSTRAINT "FK_129efedcb305c80256db2d57a59"`) await db.query(`ALTER TABLE "event" DROP CONSTRAINT "FK_83cf1bd59aa4521ed882fa51452"`) @@ -202,8 +230,6 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "cert" DROP CONSTRAINT "FK_262e29ab91c8ebc727cc518f2fb"`) await db.query(`ALTER TABLE "cert" DROP CONSTRAINT "FK_ad35ca166ad24ecea43d7ebfca9"`) await db.query(`ALTER TABLE "cert" DROP CONSTRAINT "FK_5fbefe3a497e898aff45e44f504"`) - await db.query(`ALTER TABLE "smith_cert" DROP CONSTRAINT "FK_ae67cbd087fcea0e1ec2f70cd04"`) - await db.query(`ALTER TABLE "smith_cert" DROP CONSTRAINT "FK_5e414c1d12af16165881a16b638"`) await db.query(`ALTER TABLE "membership_event" DROP CONSTRAINT "FK_fbbd75d84ab6cc2aafeaf37a03f"`) await db.query(`ALTER TABLE "membership_event" DROP CONSTRAINT "FK_cebe39c6f75881fb55dbe137418"`) await db.query(`ALTER TABLE "change_owner_key" DROP CONSTRAINT "FK_af577baa612d86d98a1ae583438"`) @@ -211,6 +237,7 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "change_owner_key" DROP CONSTRAINT "FK_9a9f8cdcae54d8b4375d70fe8be"`) await db.query(`ALTER TABLE "ud_history" DROP CONSTRAINT "FK_a481d5b96ba0e6d8237dc733a31"`) await db.query(`ALTER TABLE "identity" DROP CONSTRAINT "FK_bafa9e6c71c3f69cef6602a8095"`) + await db.query(`ALTER TABLE "identity" DROP CONSTRAINT "FK_29e3a9afec095a892e1a3bddd39"`) await db.query(`ALTER TABLE "identity" DROP CONSTRAINT "FK_666fdfaf3a12c96e95ab0a0b31c"`) await db.query(`ALTER TABLE "tx_comment" DROP CONSTRAINT "FK_39157ef296298a13cb591f8abab"`) await db.query(`ALTER TABLE "tx_comment" DROP CONSTRAINT "FK_7c33e1189dfd717140393b2e30d"`) @@ -219,6 +246,11 @@ module.exports = class Data1717671693690 { await db.query(`ALTER TABLE "transfer" DROP CONSTRAINT "FK_2a4e1dce9f72514cd28f554ee2d"`) await db.query(`ALTER TABLE "transfer" DROP CONSTRAINT "FK_76cf30dd4464ed95ad44c5bb616"`) await db.query(`ALTER TABLE "account" DROP CONSTRAINT "FK_73d14d249a6dcf9abe42eaa6573"`) + await db.query(`ALTER TABLE "smith_cert" DROP CONSTRAINT "FK_ae67cbd087fcea0e1ec2f70cd04"`) + await db.query(`ALTER TABLE "smith_cert" DROP CONSTRAINT "FK_5e414c1d12af16165881a16b638"`) + await db.query(`ALTER TABLE "smith_event" DROP CONSTRAINT "FK_8375d29344115baadd964eca6ab"`) + await db.query(`ALTER TABLE "smith_event" DROP CONSTRAINT "FK_52efc3145aa64c6cef0dfc05a7c"`) + await db.query(`ALTER TABLE "smith" DROP CONSTRAINT "FK_b7aab883baf637058d84fb30335"`) await db.query(`ALTER TABLE "universal_dividend" DROP CONSTRAINT "FK_ad1b348c17129da2cc28fceb830"`) await db.query(`ALTER TABLE "ud_reeval" DROP CONSTRAINT "FK_38a22f60573ad32c42ca0c2c111"`) } diff --git a/db/migrations/1717671813708-EnumsMigration.js b/db/migrations/1725878575142-EnumsMigration.js similarity index 72% rename from db/migrations/1717671813708-EnumsMigration.js rename to db/migrations/1725878575142-EnumsMigration.js index 31393235101824fdf5baef9d02ec41b91254d222..505d5a30b8fe52d5c429a6f0b2ff96e0ec3acc37 100644 --- a/db/migrations/1717671813708-EnumsMigration.js +++ b/db/migrations/1725878575142-EnumsMigration.js @@ -1,7 +1,7 @@ const fs = require("fs"); -module.exports = class EnumsMigration1717671813708 { - name = "EnumsMigration1717671813708"; +module.exports = class EnumsMigration1725878575142 { + name = "EnumsMigration1725878575142"; async up(db) { await db.query(fs.readFileSync("assets/sql/EnumsMigration_up.sql", "utf8")); diff --git a/db/migrations/1717671813708-udHistoryFunction.js b/db/migrations/1725878575142-udHistoryFunction.js similarity index 71% rename from db/migrations/1717671813708-udHistoryFunction.js rename to db/migrations/1725878575142-udHistoryFunction.js index 0dcd77c78135462b868c8cf9c3ab8d312c3da619..6d5b0712d18e6b251647eb27162bb75b67493914 100644 --- a/db/migrations/1717671813708-udHistoryFunction.js +++ b/db/migrations/1725878575142-udHistoryFunction.js @@ -1,7 +1,7 @@ const fs = require("fs"); -module.exports = class udHistoryFunction1717671813708 { - name = "udHistoryFunction1717671813708"; +module.exports = class udHistoryFunction1725878575142 { + name = "udHistoryFunction1725878575142"; async up(db) { await db.query(fs.readFileSync("assets/sql/udHistoryFunction_up.sql", "utf8")); diff --git a/hasura/metadata/databases/default/tables/public_account.yaml b/hasura/metadata/databases/default/tables/public_account.yaml index eccd1ef5f4c30b8b6ae0b7a75c7def53f78f91db..a78207b30d34eac1ea54072f85e5abffd8c11669 100644 --- a/hasura/metadata/databases/default/tables/public_account.yaml +++ b/hasura/metadata/databases/default/tables/public_account.yaml @@ -13,6 +13,13 @@ object_relationships: using: foreign_key_constraint_on: linked_identity_id array_relationships: + - name: removedIdentities + using: + foreign_key_constraint_on: + column: account_removed_id + table: + name: identity + schema: public - name: wasIdentity using: foreign_key_constraint_on: diff --git a/hasura/metadata/databases/default/tables/public_enum_smith_event_type.yaml b/hasura/metadata/databases/default/tables/public_enum_smith_event_type.yaml new file mode 100644 index 0000000000000000000000000000000000000000..817feed65cf0ce79d09c6503bc51485ed243db36 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_enum_smith_event_type.yaml @@ -0,0 +1,4 @@ +table: + schema: public + name: smith_event_type +is_enum: true diff --git a/hasura/metadata/databases/default/tables/public_identity.yaml b/hasura/metadata/databases/default/tables/public_identity.yaml index fb99076b36abde9bf5daf26bb8e3fd70f3de9541..599877684f0c5a8111fefe346039c42714634085 100644 --- a/hasura/metadata/databases/default/tables/public_identity.yaml +++ b/hasura/metadata/databases/default/tables/public_identity.yaml @@ -5,9 +5,19 @@ object_relationships: - name: account using: foreign_key_constraint_on: account_id + - name: accountRemoved + using: + foreign_key_constraint_on: account_removed_id - name: createdIn using: foreign_key_constraint_on: created_in_id + - name: smith + using: + foreign_key_constraint_on: + column: identity_id + table: + name: smith + schema: public array_relationships: - name: certIssued using: @@ -23,20 +33,6 @@ array_relationships: table: name: cert schema: public - - name: smithCertIssued - using: - foreign_key_constraint_on: - column: issuer_id - table: - name: smith_cert - schema: public - - name: smithCertReceived - using: - foreign_key_constraint_on: - column: receiver_id - table: - name: smith_cert - schema: public - name: membershipHistory using: foreign_key_constraint_on: diff --git a/hasura/metadata/databases/default/tables/public_population_history.yaml b/hasura/metadata/databases/default/tables/public_population_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..997906311d63c2e14c002f1c20b3657cd8f4e846 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_population_history.yaml @@ -0,0 +1,11 @@ +table: + name: population_history + schema: public +object_relationships: [] +array_relationships: [] +select_permissions: + - role: public + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/hasura/metadata/databases/default/tables/public_smith.yaml b/hasura/metadata/databases/default/tables/public_smith.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f1315b6a64882afc524bee9018a12018c932df7 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_smith.yaml @@ -0,0 +1,35 @@ +table: + name: smith + schema: public +object_relationships: + - name: identity + using: + foreign_key_constraint_on: identity_id +array_relationships: + - name: smithCertIssued + using: + foreign_key_constraint_on: + column: issuer_id + table: + name: smith_cert + schema: public + - name: smithCertReceived + using: + foreign_key_constraint_on: + column: receiver_id + table: + name: smith_cert + schema: public + - name: smithHistory + using: + foreign_key_constraint_on: + column: smith_id + table: + name: smith_event + schema: public +select_permissions: + - role: public + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/hasura/metadata/databases/default/tables/public_smith_event.yaml b/hasura/metadata/databases/default/tables/public_smith_event.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21d9c886d2c868aba189d71a39aaf5758765701b --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_smith_event.yaml @@ -0,0 +1,17 @@ +table: + name: smith_event + schema: public +object_relationships: + - name: smith + using: + foreign_key_constraint_on: smith_id + - name: event + using: + foreign_key_constraint_on: event_id +array_relationships: [] +select_permissions: + - role: public + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/hasura/metadata/databases/default/tables/public_state.yaml b/hasura/metadata/databases/default/tables/public_state.yaml new file mode 100644 index 0000000000000000000000000000000000000000..485b0d3f049a4f7c96574d5222c525e2b4434208 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_state.yaml @@ -0,0 +1,11 @@ +table: + name: state + schema: public +object_relationships: [] +array_relationships: [] +select_permissions: + - role: public + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/hasura/metadata/databases/default/tables/public_validator.yaml b/hasura/metadata/databases/default/tables/public_validator.yaml new file mode 100644 index 0000000000000000000000000000000000000000..358f72acf3abba6c0de63dae041363b023f24620 --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_validator.yaml @@ -0,0 +1,11 @@ +table: + name: validator + schema: public +object_relationships: [] +array_relationships: [] +select_permissions: + - role: public + permission: + columns: '*' + filter: {} + allow_aggregations: true diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml index bcf22e6f0ac12129be599127eb9c5e520f7a705c..78d169e1718b1cb7e2cab1046734cc198ab87a7d 100644 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ b/hasura/metadata/databases/default/tables/tables.yaml @@ -6,19 +6,24 @@ - "!include public_enum_item_type.yaml" - "!include public_enum_counter_level.yaml" - "!include public_account.yaml" +- "!include public_validator.yaml" - "!include public_transfer.yaml" +- "!include public_smith.yaml" +- "!include public_enum_smith_status.yaml" - "!include public_identity.yaml" - "!include public_enum_identity_status.yaml" -- "!include public_enum_smith_status.yaml" - "!include public_change_owner_key.yaml" - "!include public_cert.yaml" - "!include public_cert_event.yaml" - "!include public_enum_event_type.yaml" - "!include public_smith_cert.yaml" +- "!include public_smith_event.yaml" +- "!include public_enum_smith_event_type.yaml" - "!include public_membership_event.yaml" - "!include public_universal_dividend.yaml" - "!include public_ud_reeval.yaml" - "!include public_ud_history.yaml" - "!include public_tx_comment.yaml" - "!include public_enum_comment_type.yaml" +- "!include public_population_history.yaml" - "!include public_migrations.yaml" \ No newline at end of file diff --git a/schema.graphql b/schema.graphql index 7818f8ab52b263c007cffe555c997fb354ccc20e..d39e15d78ffb6129171715722f5a0e15d4219aec 100644 --- a/schema.graphql +++ b/schema.graphql @@ -116,6 +116,9 @@ type Account @entity { id: ID! "current account for the identity" identity: Identity @derivedFrom(field: "account") + "removed identities on this account" + # they are handled apart to avoid dropping the @unique constraint of account + removedIdentities: [Identity] @derivedFrom(field: "accountRemoved") "was once account of the identity" wasIdentity: [ChangeOwnerKey!] @derivedFrom(field: "previous") # there should be at most one "linked to the identity" @@ -126,6 +129,19 @@ type Account @entity { transfersReceived: [Transfer!] @derivedFrom(field: "to") "comments issued" commentsIssued: [TxComment!] @derivedFrom(field: "author") + "is currently active" + isActive: Boolean! +} + +"Since identities can change owner key, validator table helps track which smith is forging the block" +# this could be removed if we fix the following issues +# https://git.duniter.org/nodes/rust/duniter-v2s/-/issues/197 +# https://git.duniter.org/nodes/rust/duniter-v2s/-/issues/245 +type Validator @entity { + "SS58 of pubkey used at least once to compute a block" + id: ID! + "Identity index of Smith who owned this pubkey" + index: Int! @index } "Transfers" @@ -146,6 +162,26 @@ type Transfer @entity { comment: TxComment } +type Smith @entity { + "Identity index" + index: Int! @index @unique + identity: Identity! @unique + "Smith certifications issued" + smithCertIssued: [SmithCert!] @derivedFrom(field: "issuer") + "Smith certifications received" + smithCertReceived: [SmithCert!] @derivedFrom(field: "receiver") + "Smith status of the identity" + smithStatus: SmithStatus + "history of the smith changes events" + smithHistory: [SmithEvent] @derivedFrom(field: "smith") + "Last status change" + lastChanged: Int + "Number of forged blocks" + forged: Int! + "Last forged block" + lastForged: Int +} + # === this part of the schema is for Duniter pallets === # @@ -154,12 +190,17 @@ type Identity @entity { "Identity index" index: Int! @index @unique "Current account" - account: Account! @unique + # should be null for a removed identity and only in this case + account: Account @unique + "Let track the account in case identity was removed" + # should be non null for a removed identity and only in this case + accountRemoved: Account "Name" name: String! @index "Status of the identity" # mixes identity pallet status (Unconfirmed, Unvalidated, Revoked) # and membership pallet status (Member, WasMember) + # and a special status "Removed" specific to the indexer status: IdentityStatus! @index "Block number of identity creation event" createdOn: Int! @@ -167,18 +208,10 @@ type Identity @entity { createdIn: Event! "Block number of last identity, changeOwnerKey and membership event" lastChangeOn: Int! - "Smith status of the identity" - # no need for smith membership history for this one - # nullable field defaulting to null because most identity are not smith - smithStatus: SmithStatus "Certifications issued" certIssued: [Cert!] @derivedFrom(field: "issuer") "Certifications received" certReceived: [Cert!] @derivedFrom(field: "receiver") - "Smith certifications issued" - smithCertIssued: [SmithCert!] @derivedFrom(field: "issuer") - "Smith certifications received" - smithCertReceived: [SmithCert!] @derivedFrom(field: "receiver") "True if the identity is a member" isMember: Boolean! "the current expireOn value" @@ -191,6 +224,8 @@ type Identity @entity { ownerKeyChange: [ChangeOwnerKey!] @derivedFrom(field: "identity") "linked accounts" linkedAccount: [Account!] @derivedFrom(field: "linkedIdentity") + "Smith information" + smith: Smith @derivedFrom(field: "identity") "Universal Dividend history" udHistory: [UdHistory!] @derivedFrom(field: "identity") } @@ -252,11 +287,25 @@ type CertEvent @entity { "Smith certification" type SmithCert @entity { - issuer: Identity! - receiver: Identity! + issuer: Smith! + receiver: Smith! createdOn: Int! } +type SmithEvent @entity { + smith: Smith! @index + eventType: SmithEventType! + event: Event! + blockNumber: Int! @index +} + +enum SmithEventType { + Invited + Accepted + Promoted + Excluded +} + type MembershipEvent @entity { identity: Identity! @index eventType: EventType! @@ -333,4 +382,16 @@ enum CommentType { Unicode "no known type, raw bytes" Raw -} \ No newline at end of file +} + +"History of the blockchain population." +type PopulationHistory @entity { + "The count of smiths at this point in the history." + smithCount: Int! + "The count of members at this point in the history." + memberCount: Int! + "The count of active accounts at this point in the history." + activeAccountCount: Int! + "The block number at which this history record was created." + blockNumber: Int! @index @unique +} diff --git a/src/data_handler.ts b/src/data_handler.ts index 847ed31eb586f0707f5eef9cf8135745aebb45d3..3b8918e0f26025fd97a7a6ffe2e5da23ace20c80 100644 --- a/src/data_handler.ts +++ b/src/data_handler.ts @@ -10,14 +10,19 @@ import { Identity, IdentityStatus, MembershipEvent, + SmithEvent, + SmithEventType, + Smith, SmithCert, SmithStatus, + PopulationHistory, Transfer, TxComment, UdReeval, UniversalDividend, + Validator, } from "./model"; -import { Address, Ctx, Data, IdtyIndex, NewData } from "./types_custom"; +import { Address, BlockNumber, Ctx, Data, IdtyIndex, NewData } from "./types_custom"; import { hexToString } from "./utils"; import { events } from "./types"; import { getCommentType } from "./comment"; @@ -29,7 +34,11 @@ export class DataHandler { this.data = { accounts: new Map(), identities: new Map(), + smiths: new Map(), + populationHistories: [], + validators: new Map(), membershipEvents: [], + smithEvents: [], changeOwnerKey: [], transfers: new Map(), certification: new Map(), @@ -42,13 +51,39 @@ export class DataHandler { } async processNewData(newData: NewData, ctx: Ctx) { + + // Process population history by adding the value + // of the last point in database. + if (newData.populationHistories) { + const lastHistory = await ctx.store.findOneOrFail(PopulationHistory, { + where: {}, + order: { blockNumber: 'DESC' } + }); + + this.data.populationHistories = newData.populationHistories.map(history => (new PopulationHistory({ + activeAccountCount: history.activeAccountCount + lastHistory.activeAccountCount, + memberCount: history.memberCount + lastHistory.memberCount, + smithCount: history.smithCount + lastHistory.smithCount, + blockNumber: history.blockNumber, + id: `population-${history.blockNumber}`, + }))); + } + // Process accounts for (const accountId of newData.accounts) { - const newAccount = new Account({ id: accountId }); + const newAccount = new Account({ id: accountId, isActive: true }); this.data.accounts.set(accountId, newAccount); ctx.log.info(`Added account ${accountId}`); } + // Process killed accounts + for (const accountId of newData.killedAccounts) { + const killedAccount = await this.getAccountByAddressOrFail(ctx, accountId); + killedAccount.isActive = false; + this.data.accounts.set(accountId, killedAccount); + ctx.log.info(`Killed account ${accountId}`); + } + // Process transfers for (const transfer of newData.transfers) { // const fromAccount = await this.getOrCreateAccount(ctx, transfer.from); @@ -102,11 +137,13 @@ export class DataHandler { // Process identities created for (const identity of newData.identitiesCreated) { - const account = await this.getOrCreateAccount(ctx, identity.accountId); + const account = await this.getAccountByAddressOrFail(ctx, identity.accountId); const newIdentity = new Identity({ id: identity.event.id, index: identity.index, - name: identity.event.id, // Using the id of the creation event as the name for unconfirmed identities + // Using the id of the creation event as the name for unconfirmed identities + // dirty hack to allow name to be non nullable + name: identity.event.id, status: IdentityStatus.Unconfirmed, account, isMember: false, @@ -139,10 +176,12 @@ export class DataHandler { // Process identities removed for (const identity of newData.identitiesRemoved) { - const idty = await this.getIdtyByIndexOrFail(ctx, identity.index); + const idty = await this.getIdtyWithAccountByIndexOrFail(ctx, identity.index); ctx.log.info( `Set identity ${identity.index} status to Removed for reason ${identity.reason.__kind}` ); + idty.accountRemoved = idty.account!; // we are sure that a non-removed identity has non-null account + idty.account = null; // having nullable field allows to keep the @unique constraint idty.status = IdentityStatus.Removed; idty.lastChangeOn = identity.blockNumber; idty.expireOn = identity.expireOn; @@ -170,7 +209,7 @@ export class DataHandler { new ChangeOwnerKey({ id: idtyChange.id, identity: idty, - previous: idty.account, + previous: idty.account!, // changing owner key is only possible for a non-removed identity next: new_account, blockNumber: idtyChange.blockNumber, }) @@ -180,6 +219,18 @@ export class DataHandler { idty.lastChangeOn = idtyChange.blockNumber; idty.expireOn = idtyChange.expireOn; this.data.identities.set(idtyChange.index, idty); + + // Related to https://git.duniter.org/nodes/rust/duniter-v2s/-/issues/245. + // An account can be dissociated from an identity even if the validator is online. + // To circumvent this problem, we keep a mapping between the account and the last + // associated identity. + const smith = this.data.smiths.get(idtyChange.index) ?? + await ctx.store.findOneBy(Smith, { index: idtyChange.index }); + if (smith && smith.smithStatus == SmithStatus.Smith) { + // if an active smith changed owner key, we track this with a new validator + const validator = new Validator({ id: idtyChange.accountId, index: idtyChange.index }) + this.data.validators.set(idtyChange.accountId, validator); + } } // Process membership added @@ -202,7 +253,6 @@ export class DataHandler { blockNumber: event.block.height, }) ); - this.data.identities.set(identity.index, identity); } @@ -226,7 +276,6 @@ export class DataHandler { blockNumber: event.block.height, }) ); - this.data.identities.set(identity.index, identity); } @@ -322,6 +371,17 @@ export class DataHandler { ); } + // Process validators + for (const { block, validatorId } of newData.validators) { + // validator must already exist because it was created at the same time as smith, which is necessary to compute block + const validator = await this.getValidatorByAddressOrFail(ctx, validatorId); + // we are sure that a smith exist because they are created at the same time + const smith = await this.getSmithByIndexOrFail(ctx, validator.index); + smith.forged += 1; + smith.lastForged = block + this.data.smiths.set(smith.index, smith); + } + // Process certifications removals for (const c of newData.certRemoval) { const { issuerId, receiverId, blockNumber, event } = c; @@ -356,8 +416,8 @@ export class DataHandler { where: { issuer: { index: issuerId }, receiver: { index: receiverId } }, }); if (cert == null) { - const issuer = await this.getIdtyByIndexOrFail(ctx, issuerId); - const receiver = await this.getIdtyByIndexOrFail(ctx, receiverId); + const issuer = await this.getSmithByIndexOrFail(ctx, issuerId); + const receiver = await this.getSmithByIndexOrFail(ctx, receiverId); cert = new SmithCert({ id, issuer, @@ -381,39 +441,84 @@ export class DataHandler { // Process Smith invitation sent for (const invitedSmith of newData.smithInvited) { - const { idtyIndex } = invitedSmith; - const identity = await this.getIdtyByIndexOrFail(ctx, idtyIndex); - identity.smithStatus = SmithStatus.Invited; - - this.data.identities.set(idtyIndex, identity); + const { idtyIndex, event } = invitedSmith; + + const smith = await this.getOrCreateSmith(ctx, idtyIndex); + smith.smithStatus = SmithStatus.Invited; + smith.lastChanged = event.block.height; + this.data.smiths.set(idtyIndex, smith); + + this.data.smithEvents.push( + new SmithEvent({ + id: `smith-invited-${idtyIndex}-${event.id}`, + smith, + eventType: SmithEventType.Invited, + event: await ctx.store.getOrFail(Event, event.id), + blockNumber: event.block.height, + }) + ); } // Process Smith invitation accepted for (const acceptedSmithInvitations of newData.smithAccepted) { - const { idtyIndex } = acceptedSmithInvitations; - - const identity = await this.getIdtyByIndexOrFail(ctx, idtyIndex); - identity.smithStatus = SmithStatus.Pending; - this.data.identities.set(idtyIndex, identity); + const { idtyIndex, event } = acceptedSmithInvitations; + + const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex); + smith.smithStatus = SmithStatus.Pending; + smith.lastChanged = event.block.height; + this.data.smiths.set(idtyIndex, smith); + this.data.smithEvents.push( + new SmithEvent({ + id: `smith-accepted-${idtyIndex}-${event.id}`, + smith, + eventType: SmithEventType.Accepted, + event: await ctx.store.getOrFail(Event, event.id), + blockNumber: event.block.height, + }) + ); } // Process Smith promotion for (const promotedSmith of newData.smithPromoted) { - const { idtyIndex } = promotedSmith; + const { idtyIndex, event } = promotedSmith; + + const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex); + smith.smithStatus = SmithStatus.Smith; + smith.lastChanged = event.block.height; + this.data.smiths.set(idtyIndex, smith); + this.data.smithEvents.push( + new SmithEvent({ + id: `smith-promoted-${idtyIndex}-${event.id}`, + smith, + eventType: SmithEventType.Promoted, + event: await ctx.store.getOrFail(Event, event.id), + blockNumber: event.block.height, + }) + ); - const identity = await this.getIdtyByIndexOrFail(ctx, idtyIndex); - identity.smithStatus = SmithStatus.Smith; - this.data.identities.set(idtyIndex, identity); + const identity = await this.getIdtyWithAccountByIndexOrFail(ctx, idtyIndex); + const smithAccount = identity.account!; // a smith can only be a non-removed identity + const validator = new Validator({ id: smithAccount.id, index: idtyIndex }) + this.data.validators.set(smithAccount.id, validator); } // Process Smith exlusion for (const excludedSmith of newData.smithExcluded) { - const { idtyIndex } = excludedSmith; - - const identity = await this.getIdtyByIndexOrFail(ctx, idtyIndex); - - identity.smithStatus = SmithStatus.Excluded; - this.data.identities.set(idtyIndex, identity); + const { idtyIndex, event } = excludedSmith; + + const smith = await this.getSmithByIndexOrFail(ctx, idtyIndex); + smith.smithStatus = SmithStatus.Excluded; + smith.lastChanged = event.block.height; + this.data.smiths.set(idtyIndex, smith); + this.data.smithEvents.push( + new SmithEvent({ + id: `smith-excluded-${idtyIndex}-${event.id}`, + smith, + eventType: SmithEventType.Excluded, + event: await ctx.store.getOrFail(Event, event.id), + blockNumber: event.block.height, + }) + ); } // Process account links @@ -466,7 +571,9 @@ export class DataHandler { } } - // this is a hack to handle circular dependency + // this is a hack to handle circular dependency between account and identity + // account that do not exist in database are created first so that identities can be created after + // then accounts can be modified and have linkedIdentity point to the newly created identity async handleNewAccountsApart(ctx: Ctx, newData: NewData) { // Combine account and account link sets to get unique candidates const newAccountCandidates = new Set<Address>([ @@ -490,7 +597,7 @@ export class DataHandler { // Filter and create accounts that don't already exist const accountsToCreate = [...newAccountCandidates] .filter((id) => !existingAccountIds.has(id)) - .map((id) => new Account({ id })); + .map((id) => new Account({ id, isActive: true })); if (accountsToCreate.length > 0) { await ctx.store.insert(accountsToCreate); @@ -507,21 +614,17 @@ export class DataHandler { } } - // this is a hack to handle circular dependency + // this is a hack to handle circular dependency between account and identity + // identities that are created are handled apart from the ones who only changed async handleNewIdentitiesApart(ctx: Ctx, newData: NewData) { - const identities: Array<Identity> = []; + const identities: Array<Identity> = []; // collects identities (only newly created) for (const i of newData.identitiesCreated) { const idty = this.data.identities.get(i.index); - assert( - idty != null, - "created identities must appear in prepared identities" - ); + assert(idty, "created identities must appear in prepared identities"); this.data.identities.delete(i.index); // prevent from trying to add twice identities.push(idty); } - if (identities.length === 0) { - return; - } + if (identities.length === 0) return; // we are sure that all created identities actually do not already exist in database await ctx.store.insert(identities); await ctx.store.commit(); @@ -533,9 +636,12 @@ export class DataHandler { // account can have already existed, been killed, and recreated await ctx.store.upsert([...this.data.accounts.values()]); // identities can have been changed (confirmed, change owner key...) or added (created) + await ctx.store.upsert([...this.data.smiths.values()]); await ctx.store.upsert([...this.data.identities.values()]); + await ctx.store.upsert([...this.data.validators.values()]); // membership can have been created, renewed, or removed await ctx.store.upsert([...this.data.membershipEvents.values()]); + await ctx.store.upsert([...this.data.smithEvents.values()]); // certs can have been created, renewed, or removed await ctx.store.upsert([...this.data.certification.values()]); await ctx.store.upsert([...this.data.smithCert.values()]); @@ -547,6 +653,7 @@ export class DataHandler { await ctx.store.insert(this.data.certEvent); await ctx.store.insert(this.data.universalDividend); await ctx.store.insert(this.data.udReeval); + await ctx.store.insert(this.data.populationHistories); // Apply changes in database await ctx.store.commit(); @@ -557,7 +664,7 @@ export class DataHandler { this.data.accounts.get(id) ?? (await ctx.store.get(Account, id)); if (account == null) { // we need to create it - account = new Account({ id }); + account = new Account({ id, isActive: true }); this.data.accounts.set(id, account); } return account; @@ -577,12 +684,44 @@ export class DataHandler { ); } + async getSmithByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Smith> { + return ( + this.data.smiths.get(index) ?? + ctx.store.findOneByOrFail(Smith, { index }) + ); + } + async getIdtyWithAccountByIndexOrFail(ctx: Ctx, index: IdtyIndex): Promise<Identity> { return ( this.data.identities.get(index) ?? - await ctx.store.findOneOrFail(Identity, { + ctx.store.findOneOrFail(Identity, { relations: { account: true }, where: { index: index }, })) } + + async getValidatorByAddressOrFail(ctx: Ctx, address: Address): Promise<Validator> { + return ( + this.data.validators.get(address) ?? + ctx.store.findOneByOrFail(Validator, { id: address }) + ); + } + + async getOrCreateSmith(ctx: Ctx, index: IdtyIndex): Promise<Smith> { + const smith = + this.data.smiths.get(index) ?? + await ctx.store.findOneBy(Smith, { index }); + if (smith) { + return smith; + } + else { + const identity = await this.getIdtyByIndexOrFail(ctx, index); + return new Smith({ + id: `smith_${index}`, + index: index, + identity: identity, + forged: 0, + }); + } + } } diff --git a/src/genesis/genesis.ts b/src/genesis/genesis.ts index 68a52cdabfee7e8a9b7235c44aec0b46ae87406e..25c218fe50143676d4c24f5d80dc3cd67a1d5918 100644 --- a/src/genesis/genesis.ts +++ b/src/genesis/genesis.ts @@ -1,6 +1,6 @@ import { readFileSync } from "fs"; import path from "path/posix"; -import { Account, Block, Cert, CertEvent, CommentType, Event, EventType, Identity, MembershipEvent, SmithCert, SmithStatus, Transfer, TxComment } from "../model"; +import { Account, Block, Cert, CertEvent, CommentType, Event, EventType, Smith, PopulationHistory, Identity, MembershipEvent, SmithEvent, SmithEventType, SmithCert, SmithStatus, Transfer, Validator, TxComment } from "../model"; import type { Address, BlockV1, Certv1, Ctx, Genesis, Genv1, IdtyIndex, TransactionHistory } from "../types_custom"; import { bytesToString, hexToUint8Array, safePubkeyToAddress, v1_to_v2_height } from "../utils"; import { AccountId32 } from "../types/v800"; @@ -53,8 +53,9 @@ export async function saveGenesis(ctx: Ctx, block: Block) { } // =========================================== + // === v1 block history === - ctx.log.info("Handling block history"); + ctx.log.info("Handling v1 block history"); const blocksv1: Array<BlockV1> = JSON.parse(readFileSync(path.resolve(hist_block_path)).toString()) for (const b of blocksv1) { @@ -88,9 +89,27 @@ export async function saveGenesis(ctx: Ctx, block: Block) { createFakeEvent(b.height, negBlock, negHeight) } + // create fake genesis event to mark identities as created in this block + // TODO with membership history we can use negative block instead + const genesis_event = new Event({ + id: "genesis-event_0", + index: 0, + block, + phase: 'genesis-phase', + pallet: 'genesis-pallet', + name: 'genesis-name', + }); + fakeBlockEvents.set(0, genesis_event) + + // SAVE v1 block history + ctx.log.info("Saving v1 block history"); + await ctx.store.insert([...blocks.values()]); + await ctx.store.insert([...fakeBlockEvents.values()]); + // =========================================== + // === genesis file === - ctx.log.info("Loading genesis"); + ctx.log.info("Loading genesis file"); // Read genesis json const genesisData = JSON.parse(readFileSync(path.resolve(process.cwd(), genesis_path)).toString()); @@ -99,34 +118,35 @@ export async function saveGenesis(ctx: Ctx, block: Block) { const accounts: Map<Address, Account> = new Map(); const identities: Map<IdtyIndex, Identity> = new Map(); + const validators: Map<Address, Validator> = new Map(); + const smiths: Map<IdtyIndex, Smith> = new Map(); const identitiesMap: Map<AccountId32, IdtyIndex> = new Map(); const certs: Map<string, Cert> = new Map(); const smithCerts: SmithCert[] = []; const membershipsEvents: MembershipEvent[] = []; - - - // create fake genesis event to get block height and timestamp - const genesis_event = new Event({ - id: "genesis-event_0", - index: 0, - block, - phase: 'genesis-phase', - pallet: 'genesis-pallet', - name: 'genesis-name', - }); - fakeBlockEvents.set(0, genesis_event) - - // collect accounts - for (const [address] of Object.entries(genesis.account.accounts)) { + const smithsEvents: SmithEvent[] = []; + const populationHistory = new PopulationHistory({ id: "genesis-population", blockNumber: -1, activeAccountCount: 0, memberCount: 0, smithCount: 0 }) + + // --- collect accounts --- + for (const [address, value] of Object.entries(genesis.account.accounts)) { + // Accounts with 0 will not be added in blockchain. + // Accounts with less than 1ED will be reaped at first block. + const isActive = value.balance > 0; accounts.set( address, new Account({ id: address, + isActive, + // init to null and update later to handle circular dependency + linkedIdentity: null, }) ); + if (isActive) { + populationHistory.activeAccountCount += 1; + } } - // collect identities + // --- collect identities --- for (const idty of genesis.identity.identities) { const account = accounts.get(idty.value.owner_key); const the_identity = new Identity({ @@ -147,7 +167,7 @@ export async function saveGenesis(ctx: Ctx, block: Block) { identitiesMap.set(idty.value.owner_key, idty.index); } - // collect memberships + // --- collect memberships --- for (const [idtyIndex, mshipInfo] of Object.entries(genesis.membership.memberships)) { const identity = identities.get(parseInt(idtyIndex))!; identity.isMember = true; @@ -161,9 +181,10 @@ export async function saveGenesis(ctx: Ctx, block: Block) { blockNumber: 0, }) ); + populationHistory.memberCount += 1; } - // collect certifications + // --- collect certifications --- for (const [receiver_index, certs_received] of Object.entries(genesis.certification.certsByReceiver)) { for (const [issuer_index, expiration_block] of Object.entries(certs_received)) { const certstr = `${issuer_index}-${receiver_index}` @@ -185,26 +206,85 @@ export async function saveGenesis(ctx: Ctx, block: Block) { } } - // collect smith memberships - for (const [idtyIdex, smithCertsData] of Object.entries(genesis.smithMembers.initialSmiths)) { - const [isOnline, certs] = smithCertsData as [boolean, number[]]; - const identity = identities.get(parseInt(idtyIdex))!; - identity.smithStatus = SmithStatus.Smith; - identities.set(identity.index, identity); - + // --- collect genesis smith --- + // loop once to add smith entities + for (const [idtyIndex, _] of Object.entries(genesis.smithMembers.initialSmiths)) { + const idtyIdx = parseInt(idtyIndex) + const identity = identities.get(idtyIdx)!; + + const smith = new Smith({ + id: `genesis-smith_${idtyIndex}`, + index: idtyIdx, + identity: identity, + forged: 0, + lastChanged: 0, + smithStatus: SmithStatus.Smith, + }); + smiths.set(idtyIdx, smith); + populationHistory.smithCount += 1; + + // smith must be non-removed identity with a non-null account + const validator = new Validator({ id: identity.account!.id, index: idtyIdx }) + validators.set(validator.id, validator); + + smithsEvents.push( + new SmithEvent({ + id: `genesis-smith_${idtyIndex}`, + smith: smiths.get(idtyIdx)!, + eventType: SmithEventType.Promoted, + event: genesis_event, + blockNumber: 0, + }) + ); + } + // loop another time to add smith certs + for (const [idtyIndex, smithCertsData] of Object.entries(genesis.smithMembers.initialSmiths)) { + const idtyIdx = parseInt(idtyIndex) + const [_, certs] = smithCertsData as [boolean, number[]]; for (const issuer_index of certs) { smithCerts.push( new SmithCert({ - id: `genesis-smith-membership_${issuer_index}-${idtyIdex}`, - issuer: identities.get(issuer_index), - receiver: identities.get(parseInt(idtyIdex)), + id: `genesis-smith-membership_${issuer_index}-${idtyIndex}`, + issuer: smiths.get(issuer_index)!, + receiver: smiths.get(idtyIdx)!, createdOn: 0, - }) + }), ); } } + // SAVE genesis data + ctx.log.info("Saving data from genesis file"); + + // insert everything in storage + await ctx.store.insert([...accounts.values()]); + await ctx.store.insert([...identities.values()]); + await ctx.store.insert([...certs.values()]); + + // HACK to handle circular dependency, linkedIdentity should be + // updated after being created null AND inserted in db + for (const [address, value] of Object.entries(genesis.account.accounts)) { + let account = accounts.get(address); + let identity = identities.get(parseInt(value.idty_id)); + if (account && identity) { + account.linkedIdentity = identity; + accounts.set(address, account); + } + } + // accounts are marked for update with new linked identity + await ctx.store.upsert([...accounts.values()]); + await ctx.store.insert([...smiths.values()]); + await ctx.store.insert([...validators.values()]); + // membership events only includes genesis related events + // TODO also include v1 membership history + // https://git.duniter.org/tools/py-g1-migrator/-/issues/5 + await ctx.store.insert(membershipsEvents); + await ctx.store.insert(smithsEvents); + await ctx.store.insert(smithCerts); + await ctx.store.insert(populationHistory); + // =========================================== + // === v1 cert history (after genesis file) === ctx.log.info("Process cert history"); @@ -267,8 +347,8 @@ export async function saveGenesis(ctx: Ctx, block: Block) { if (event == undefined) { // this can happen for cert removal because duniter thinks that hasEvent = false event = createFakeEvent(c.blockNumber, blocks.get(negHeight)!, negHeight) - } + // add new cert event to trace this history const certEvent = new CertEvent({ id: `genesis-cert-event_${c.issuer}-${c.receiver}_v1-${c.blockNumber}`, blockNumber: negHeight, @@ -279,6 +359,14 @@ export async function saveGenesis(ctx: Ctx, block: Block) { certEvents.push(certEvent); } + // SAVE cert events + ctx.log.info("Saving certification history"); + // add the fake events needed for these certs + await ctx.store.upsert([...fakeBlockEvents.values()]); + // add the certs needed for the history and not present in the genesis + await ctx.store.upsert([...certs.values()]); + await ctx.store.insert(certEvents); + // =========================================== // accounts present in transaction history but not in genesis @@ -291,6 +379,7 @@ export async function saveGenesis(ctx: Ctx, block: Block) { const tx_history: TransactionHistory = JSON.parse(readFileSync(path.resolve(hist_tx_path)).toString()) // make sure that all tx source and dest accounts exist so that we can get them later when adding txs + // we know that these accounts are not active anymore (otherwise they would have been added before) for (const tx of tx_history) { const issuer = safePubkeyToAddress(tx.from) const receiver = safePubkeyToAddress(tx.to) @@ -298,12 +387,14 @@ export async function saveGenesis(ctx: Ctx, block: Block) { unknown_wallet_count++ accounts.set(issuer, new Account({ id: issuer, + isActive: false, })); } if (!accounts.has(receiver)) { unknown_wallet_count++ accounts.set(receiver, new Account({ id: receiver, + isActive: false, })); } } @@ -354,25 +445,19 @@ export async function saveGenesis(ctx: Ctx, block: Block) { } // SAVE - ctx.log.info("Saving genesis"); - await ctx.store.insert([...blocks.values()]); - await ctx.store.insert([...fakeBlockEvents.values()]); - await ctx.store.insert([...accounts.values()]); - await ctx.store.insert([...identities.values()]); - await ctx.store.insert([...certs.values()]); - await ctx.store.insert(certEvents); - await ctx.store.insert(smithCerts); - await ctx.store.insert(membershipsEvents); + ctx.log.info("Saving v1 transaction history and comments"); + // adding transfers and comments requires to add accounts unknown in the genesis + await ctx.store.upsert([...accounts.values()]); await ctx.store.insert(genesis_txcomments); await ctx.store.insert(genesis_transfers); - ctx.log.info("Genesis saved"); // FLUSH ctx.log.info("Flushing changes to storage, this can take a while..."); + ctx.log.info("(about ~5 minutes for all g1 history and genesis data)"); await ctx.store.flush(); ctx.log.info("Genesis flushed"); // START ctx.log.info("====================="); - ctx.log.info("Starting blockchain indexing"); + ctx.log.info(`Starting blockchain indexing with ${populationHistory.smithCount} smiths, ${populationHistory.memberCount} members and ${populationHistory.activeAccountCount} accounts!`); } diff --git a/src/main.ts b/src/main.ts index 237eff9a8f76c4ec0a7c4d36e2e5c54646683dac..980ee46a5e37899dcb356d798acc118d5cc1d0b2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,7 +5,7 @@ import { saveGenesis } from "./genesis/genesis"; import { saveBlock, saveCall, saveEvent, saveExtrinsic } from "./giant-squid"; import { processor } from "./processor"; import { constants, events as events_t } from "./types"; -import { Ctx, NewData } from "./types_custom"; +import { Ctx, NewData, AccPopulationHistory } from "./types_custom"; import { ss58encode } from "./utils"; // main processor loop able to manage a batch of blocks @@ -38,6 +38,9 @@ processor.run(new TypeormDatabaseWithCache(), async (ctx) => { const newData: NewData = { accounts: [], + killedAccounts: [], + validators: [], + populationHistories: [], identitiesCreated: [], identitiesConfirmed: [], identitiesValidated: [], @@ -80,7 +83,6 @@ processor.run(new TypeormDatabaseWithCache(), async (ctx) => { function collectDataFromEvents(ctx: Ctx, newData: NewData) { const silence_events = [ events_t.system.extrinsicSuccess.name, - events_t.system.killedAccount.name, events_t.certification.certRemoved.name, events_t.session.newSession.name, events_t.imOnline.allGood.name, @@ -94,7 +96,18 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { events_t.quota.refunded.name, ]; + // accumulator to count population history + const acc: AccPopulationHistory = { activeAccountCount: 0, memberCount: 0, smithCount: 0, blockNumber: -1 }; + ctx.blocks.forEach((block) => { + + const validator = block.header.validator; + if (validator != null) { + // (only block 0 has null validator) + const blockNumber = block.header.height; + newData.validators.push({ block: blockNumber, validatorId: ss58encode(validator) }); + } + block.events.forEach((event) => { if (!silence_events.includes(event.name)) { ctx.log.info("" + block.header.height + " " + event.name); @@ -103,6 +116,15 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { case events_t.system.newAccount.name: { const evt = events_t.system.newAccount.v800.decode(event); newData.accounts.push(ss58encode(evt.account)); + acc.activeAccountCount += 1; + acc.blockNumber = block.header.height; + break; + } + + case events_t.system.killedAccount.name: { + const evt = events_t.system.killedAccount.v800.decode(event); + newData.killedAccounts.push(ss58encode(evt.account)); + acc.activeAccountCount -= 1; break; } @@ -119,6 +141,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { newData.accountLink.push({ accountId: ss58encode(evt.who), index: evt.identity, + event: event, }); break; } @@ -128,6 +151,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { events_t.account.accountUnlinked.v800.decode(event); newData.accountUnlink.push({ accountId: ss58encode(evt), + event: event, }); break; } @@ -145,6 +169,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { from: ss58encode(evt.from), to: ss58encode(evt.to), amount: evt.amount, + event: event, }); break; } @@ -174,6 +199,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expireOn: block.header.height + constants.identity.confirmPeriod.v800.get(event.block), + event: event, }); break; } @@ -188,6 +214,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expireOn: block.header.height + constants.identity.validationPeriod.v800.get(event.block), + event: event, }); break; } @@ -202,6 +229,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expireOn: block.header.height + constants.identity.deletionPeriod.v800.get(event.block), + event: event, }); break; } @@ -216,6 +244,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expireOn: block.header.height + constants.identity.autorevocationPeriod.v800.get(event.block), + event: event, }); break; } @@ -231,6 +260,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expireOn: block.header.height + constants.identity.changeOwnerKeyPeriod.v800.get(event.block), + event: event, }); break; } @@ -243,6 +273,8 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { expire_on: evt.expireOn, event: event, }); + acc.memberCount += 1; + acc.blockNumber = block.header.height; break; } @@ -254,6 +286,8 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { reason: evt.reason, event: event, }); + acc.memberCount -= 1; + acc.blockNumber = block.header.height; break; } @@ -317,6 +351,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { issuerId: evt.issuer, receiverId: evt.receiver, createdOn: block.header.height, + event: event, }); break; } @@ -328,6 +363,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { id: event.id, issuerId: evt.issuer, receiverId: evt.receiver, + event: event, }); break; } @@ -338,6 +374,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { newData.smithPromoted.push({ id: event.id, idtyIndex: evt.idtyIndex, + event: event, }); break; } @@ -348,7 +385,10 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { newData.smithExcluded.push({ id: event.id, idtyIndex: evt.idtyIndex, + event: event, }); + acc.smithCount += 1; + acc.blockNumber = block.header.height; break; } @@ -359,6 +399,7 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { id: event.id, idtyIndex: evt.receiver, invitedBy: evt.issuer, + event: event, }); break; } @@ -369,7 +410,10 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { newData.smithAccepted.push({ id: event.id, idtyIndex: evt.idtyIndex, + event: event, }); + acc.smithCount -= 1; + acc.blockNumber = block.header.height; break; } @@ -406,5 +450,11 @@ function collectDataFromEvents(ctx: Ctx, newData: NewData) { break; } }); + + // If the accumulator updated this block + // push it in the history. + if (acc.blockNumber == block.header.height) { + newData.populationHistories.push({ ...acc }); + } }); } diff --git a/src/model/generated/_smithEventType.ts b/src/model/generated/_smithEventType.ts new file mode 100644 index 0000000000000000000000000000000000000000..f5a3df95dd8e3fc8fe25124a5bfae7ed5513d5d1 --- /dev/null +++ b/src/model/generated/_smithEventType.ts @@ -0,0 +1,6 @@ +export enum SmithEventType { + Invited = "Invited", + Accepted = "Accepted", + Promoted = "Promoted", + Excluded = "Excluded", +} diff --git a/src/model/generated/account.model.ts b/src/model/generated/account.model.ts index 4a1d3b49c9946c734c0ab84918bc4d7336699d7f..5a48919a476d4fe828568181dc95e9f4e2541292 100644 --- a/src/model/generated/account.model.ts +++ b/src/model/generated/account.model.ts @@ -23,6 +23,12 @@ export class Account { * current account for the identity */ + /** + * removed identities on this account + */ + @OneToMany_(() => Identity, e => e.accountRemoved) + removedIdentities!: Identity[] + /** * was once account of the identity */ @@ -53,4 +59,10 @@ export class Account { */ @OneToMany_(() => TxComment, e => e.author) commentsIssued!: TxComment[] + + /** + * is currently active + */ + @Column_("bool", {nullable: false}) + isActive!: boolean } diff --git a/src/model/generated/identity.model.ts b/src/model/generated/identity.model.ts index 618992150c1242a6016f91bae6f55cbec66a72a1..3fa625b521dc275ba6e49f0a4220626333e66bcd 100644 --- a/src/model/generated/identity.model.ts +++ b/src/model/generated/identity.model.ts @@ -2,11 +2,10 @@ import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, I import {Account} from "./account.model" import {IdentityStatus} from "./_identityStatus" import {Event} from "./event.model" -import {SmithStatus} from "./_smithStatus" import {Cert} from "./cert.model" -import {SmithCert} from "./smithCert.model" import {MembershipEvent} from "./membershipEvent.model" import {ChangeOwnerKey} from "./changeOwnerKey.model" +import {Smith} from "./smith.model" import {UdHistory} from "./udHistory.model" /** @@ -34,7 +33,14 @@ export class Identity { @Index_({unique: true}) @OneToOne_(() => Account, {nullable: true}) @JoinColumn_() - account!: Account + account!: Account | undefined | null + + /** + * Let track the account in case identity was removed + */ + @Index_() + @ManyToOne_(() => Account, {nullable: true}) + accountRemoved!: Account | undefined | null /** * Name @@ -69,12 +75,6 @@ export class Identity { @Column_("int4", {nullable: false}) lastChangeOn!: number - /** - * Smith status of the identity - */ - @Column_("varchar", {length: 8, nullable: true}) - smithStatus!: SmithStatus | undefined | null - /** * Certifications issued */ @@ -87,18 +87,6 @@ export class Identity { @OneToMany_(() => Cert, e => e.receiver) certReceived!: Cert[] - /** - * Smith certifications issued - */ - @OneToMany_(() => SmithCert, e => e.issuer) - smithCertIssued!: SmithCert[] - - /** - * Smith certifications received - */ - @OneToMany_(() => SmithCert, e => e.receiver) - smithCertReceived!: SmithCert[] - /** * True if the identity is a member */ @@ -129,6 +117,10 @@ export class Identity { @OneToMany_(() => Account, e => e.linkedIdentity) linkedAccount!: Account[] + /** + * Smith information + */ + /** * Universal Dividend history */ diff --git a/src/model/generated/index.ts b/src/model/generated/index.ts index f89a685acdbf883885f826c9bfa93790de9a79b2..9ff37ed27620f600ac93840b6237a1904895b169 100644 --- a/src/model/generated/index.ts +++ b/src/model/generated/index.ts @@ -7,18 +7,23 @@ export * from "./itemsCounter.model" export * from "./_itemType" export * from "./_counterLevel" export * from "./account.model" +export * from "./validator.model" export * from "./transfer.model" +export * from "./smith.model" +export * from "./_smithStatus" export * from "./identity.model" export * from "./_identityStatus" -export * from "./_smithStatus" export * from "./changeOwnerKey.model" export * from "./cert.model" export * from "./certEvent.model" export * from "./_eventType" export * from "./smithCert.model" +export * from "./smithEvent.model" +export * from "./_smithEventType" export * from "./membershipEvent.model" export * from "./universalDividend.model" export * from "./udReeval.model" export * from "./udHistory.model" export * from "./txComment.model" export * from "./_commentType" +export * from "./populationHistory.model" diff --git a/src/model/generated/populationHistory.model.ts b/src/model/generated/populationHistory.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..58b751c796adb455892e03dc63a1f07d65fb08c8 --- /dev/null +++ b/src/model/generated/populationHistory.model.ts @@ -0,0 +1,39 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_} from "typeorm" + +/** + * History of the blockchain population. + */ +@Entity_() +export class PopulationHistory { + constructor(props?: Partial<PopulationHistory>) { + Object.assign(this, props) + } + + @PrimaryColumn_() + id!: string + + /** + * The count of smiths at this point in the history. + */ + @Column_("int4", {nullable: false}) + smithCount!: number + + /** + * The count of members at this point in the history. + */ + @Column_("int4", {nullable: false}) + memberCount!: number + + /** + * The count of active accounts at this point in the history. + */ + @Column_("int4", {nullable: false}) + activeAccountCount!: number + + /** + * The block number at which this history record was created. + */ + @Index_({unique: true}) + @Column_("int4", {nullable: false}) + blockNumber!: number +} diff --git a/src/model/generated/smith.model.ts b/src/model/generated/smith.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..6c41231affeb2662c806e55afbff06c729d0d0cc --- /dev/null +++ b/src/model/generated/smith.model.ts @@ -0,0 +1,69 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_, OneToOne as OneToOne_, JoinColumn as JoinColumn_, OneToMany as OneToMany_} from "typeorm" +import {Identity} from "./identity.model" +import {SmithCert} from "./smithCert.model" +import {SmithStatus} from "./_smithStatus" +import {SmithEvent} from "./smithEvent.model" + +@Entity_() +export class Smith { + constructor(props?: Partial<Smith>) { + Object.assign(this, props) + } + + @PrimaryColumn_() + id!: string + + /** + * Identity index + */ + @Index_({unique: true}) + @Column_("int4", {nullable: false}) + index!: number + + @Index_({unique: true}) + @OneToOne_(() => Identity, {nullable: true}) + @JoinColumn_() + identity!: Identity + + /** + * Smith certifications issued + */ + @OneToMany_(() => SmithCert, e => e.issuer) + smithCertIssued!: SmithCert[] + + /** + * Smith certifications received + */ + @OneToMany_(() => SmithCert, e => e.receiver) + smithCertReceived!: SmithCert[] + + /** + * Smith status of the identity + */ + @Column_("varchar", {length: 8, nullable: true}) + smithStatus!: SmithStatus | undefined | null + + /** + * history of the smith changes events + */ + @OneToMany_(() => SmithEvent, e => e.smith) + smithHistory!: SmithEvent[] + + /** + * Last status change + */ + @Column_("int4", {nullable: true}) + lastChanged!: number | undefined | null + + /** + * Number of forged blocks + */ + @Column_("int4", {nullable: false}) + forged!: number + + /** + * Last forged block + */ + @Column_("int4", {nullable: true}) + lastForged!: number | undefined | null +} diff --git a/src/model/generated/smithCert.model.ts b/src/model/generated/smithCert.model.ts index 1322934d69efbe6cac09ca4553694f1f9499d25d..469fe60c3c34101060e7f341280d9b29a43808ff 100644 --- a/src/model/generated/smithCert.model.ts +++ b/src/model/generated/smithCert.model.ts @@ -1,5 +1,5 @@ import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, ManyToOne as ManyToOne_, Index as Index_} from "typeorm" -import {Identity} from "./identity.model" +import {Smith} from "./smith.model" /** * Smith certification @@ -14,12 +14,12 @@ export class SmithCert { id!: string @Index_() - @ManyToOne_(() => Identity, {nullable: true}) - issuer!: Identity + @ManyToOne_(() => Smith, {nullable: true}) + issuer!: Smith @Index_() - @ManyToOne_(() => Identity, {nullable: true}) - receiver!: Identity + @ManyToOne_(() => Smith, {nullable: true}) + receiver!: Smith @Column_("int4", {nullable: false}) createdOn!: number diff --git a/src/model/generated/smithEvent.model.ts b/src/model/generated/smithEvent.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..c924daff973e47bd6de73eb03b2a9db1439c27bb --- /dev/null +++ b/src/model/generated/smithEvent.model.ts @@ -0,0 +1,29 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, ManyToOne as ManyToOne_, Index as Index_} from "typeorm" +import {Smith} from "./smith.model" +import {SmithEventType} from "./_smithEventType" +import {Event} from "./event.model" + +@Entity_() +export class SmithEvent { + constructor(props?: Partial<SmithEvent>) { + Object.assign(this, props) + } + + @PrimaryColumn_() + id!: string + + @Index_() + @ManyToOne_(() => Smith, {nullable: true}) + smith!: Smith + + @Column_("varchar", {length: 8, nullable: false}) + eventType!: SmithEventType + + @Index_() + @ManyToOne_(() => Event, {nullable: true}) + event!: Event + + @Index_() + @Column_("int4", {nullable: false}) + blockNumber!: number +} diff --git a/src/model/generated/validator.model.ts b/src/model/generated/validator.model.ts new file mode 100644 index 0000000000000000000000000000000000000000..c0ed92fb375f779e4b31bb1183da25806c779b8b --- /dev/null +++ b/src/model/generated/validator.model.ts @@ -0,0 +1,24 @@ +import {Entity as Entity_, Column as Column_, PrimaryColumn as PrimaryColumn_, Index as Index_} from "typeorm" + +/** + * Since identities can change owner key, validator table helps track which smith is forging the block + */ +@Entity_() +export class Validator { + constructor(props?: Partial<Validator>) { + Object.assign(this, props) + } + + /** + * SS58 of pubkey used at least once to compute a block + */ + @PrimaryColumn_() + id!: string + + /** + * Identity index of Smith who owned this pubkey + */ + @Index_() + @Column_("int4", {nullable: false}) + index!: number +} diff --git a/src/types_custom.ts b/src/types_custom.ts index 1f83f05d6831294667c73ee950a9eb4f636046c0..e1c6005acb7e33ba823bcb1c997eabdc227d2f82 100644 --- a/src/types_custom.ts +++ b/src/types_custom.ts @@ -7,17 +7,21 @@ import { Identity, IdentityStatus, MembershipEvent, + Smith, + SmithEvent, SmithCert, + PopulationHistory, Transfer, TxComment, UdReeval, UniversalDividend, + Validator, } from "./model"; import { Call, Event, Extrinsic, ProcessorContext } from "./processor"; import { AccountId32, MembershipRemovalReason, RemovalReason, RevocationReason } from "./types/v800"; // type aliases -type BlockNumber = number; +export type BlockNumber = number; export type Address = string; export type B58Pubkey = string; export type IdtyIndex = number; @@ -152,7 +156,11 @@ export interface Certv1 { export interface Data { accounts: Map<Address, Account>; identities: Map<IdtyIndex, Identity>; + validators: Map<Address, Validator>; + populationHistories: PopulationHistory[]; + smiths: Map<IdtyIndex, Smith>; membershipEvents: MembershipEvent[]; + smithEvents: SmithEvent[]; changeOwnerKey: ChangeOwnerKey[]; transfers: Map<string, Transfer>; certification: Map<[IdtyIndex, IdtyIndex], Cert>; @@ -169,6 +177,9 @@ export interface Data { // this contains partial data to be turned into types export interface NewData { accounts: Address[]; + killedAccounts: Address[]; + validators: BlockValidator[]; + populationHistories: AccPopulationHistory[]; identitiesCreated: IdtyCreatedEvent[]; identitiesConfirmed: IdtyConfirmedEvent[]; identitiesValidated: IdtyValidatedEvent[]; @@ -195,6 +206,13 @@ export interface NewData { comments: CommentEvents[]; } +export interface AccPopulationHistory { + activeAccountCount: number; + memberCount: number; + smithCount: number; + blockNumber: BlockNumber; +} + // id is always the id of the creation event. Only present if event is absent interface UniversalDividendEvent { blockNumber: BlockNumber; @@ -221,6 +239,7 @@ interface TransferEvent { from: Address; to: Address; amount: bigint; + event: Event; } interface CommentEvents { @@ -253,9 +272,15 @@ interface CertRemovalEvent { event: Event; } +interface BlockValidator { + block: BlockNumber; + validatorId: Address; +} + interface SmithPromotedEvent { id: string; idtyIndex: IdtyIndex; + event: Event; } interface SmithCertAddedEvent { @@ -263,28 +288,33 @@ interface SmithCertAddedEvent { issuerId: IdtyIndex; receiverId: IdtyIndex; createdOn: BlockNumber; + event: Event; } interface SmithCertRemovedEvent { id: string; issuerId: IdtyIndex; receiverId: IdtyIndex; + event: Event; } interface SmithExcludedEvent { id: string; idtyIndex: IdtyIndex; + event: Event; } interface SmithInvitedEvent { id: string; idtyIndex: IdtyIndex; invitedBy: IdtyIndex; + event: Event; } interface SmithAcceptedEvent { id: string; idtyIndex: IdtyIndex; + event: Event; } interface IdtyCreatedEvent { @@ -301,6 +331,7 @@ interface IdtyConfirmedEvent { name: string; blockNumber: BlockNumber; expireOn: BlockNumber; + event: Event; } interface IdtyValidatedEvent { @@ -308,6 +339,7 @@ interface IdtyValidatedEvent { index: IdtyIndex; blockNumber: BlockNumber; expireOn: BlockNumber; + event: Event; } interface IdtyRemovedEvent { @@ -316,6 +348,7 @@ interface IdtyRemovedEvent { reason: RemovalReason; blockNumber: BlockNumber; expireOn: BlockNumber; + event: Event; } interface IdtyRevokedEvent { @@ -324,6 +357,7 @@ interface IdtyRevokedEvent { reason: RevocationReason; blockNumber: BlockNumber; expireOn: BlockNumber; + event: Event; } interface IdtyChangedOwnerKeyEvent { @@ -332,6 +366,7 @@ interface IdtyChangedOwnerKeyEvent { accountId: Address; blockNumber: BlockNumber; expireOn: BlockNumber; + event: Event; } interface MembershipAddedEvent { @@ -355,10 +390,12 @@ interface MembershipRenewedEvent { interface AccountLinkEvent { accountId: Address; index: IdtyIndex; + event: Event; } interface AccountUnlinkEvent { accountId: Address; + event: Event; } // =========================== Hasura metadata script =========================== //