From 42fedf0a38b35c8131ea658de2090b64c9cf64ba Mon Sep 17 00:00:00 2001
From: Hugo Trentesaux <hugo@trentesaux.fr>
Date: Thu, 21 Nov 2024 16:19:29 +0100
Subject: [PATCH] add user fs
and remove tx comment
---
.../default/tables/public_user_fs.yaml | 13 +++++
.../databases/default/tables/tables.yaml | 2 +-
.../default/1732201209408_squashed/down.sql | 14 ++++++
.../default/1732201209408_squashed/up.sql | 12 +++++
src/consts.ts | 19 +++++---
src/indexer/database.ts | 48 +++++++++++++++++--
6 files changed, 98 insertions(+), 10 deletions(-)
create mode 100644 hasura/metadata/databases/default/tables/public_user_fs.yaml
create mode 100644 hasura/migrations/default/1732201209408_squashed/down.sql
create mode 100644 hasura/migrations/default/1732201209408_squashed/up.sql
diff --git a/hasura/metadata/databases/default/tables/public_user_fs.yaml b/hasura/metadata/databases/default/tables/public_user_fs.yaml
new file mode 100644
index 0000000..a068365
--- /dev/null
+++ b/hasura/metadata/databases/default/tables/public_user_fs.yaml
@@ -0,0 +1,13 @@
+table:
+ name: user_fs
+ schema: public
+select_permissions:
+ - role: public
+ permission:
+ columns:
+ - index_request_cid
+ - data_cid
+ - pubkey
+ - time
+ filter: {}
+ comment: ""
diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml
index 67e57bf..0dda00b 100644
--- a/hasura/metadata/databases/default/tables/tables.yaml
+++ b/hasura/metadata/databases/default/tables/tables.yaml
@@ -1,3 +1,3 @@
- "!include public_meta.yaml"
- "!include public_profiles.yaml"
-- "!include public_transaction_comments.yaml"
+- "!include public_user_fs.yaml"
diff --git a/hasura/migrations/default/1732201209408_squashed/down.sql b/hasura/migrations/default/1732201209408_squashed/down.sql
new file mode 100644
index 0000000..d172596
--- /dev/null
+++ b/hasura/migrations/default/1732201209408_squashed/down.sql
@@ -0,0 +1,14 @@
+
+comment on column "public"."user_fs"."time" is NULL;
+
+comment on column "public"."user_fs"."data_cid" is NULL;
+
+comment on column "public"."user_fs"."index_request_cid" is NULL;
+
+comment on column "public"."user_fs"."pubkey" is NULL;
+
+DROP TABLE "public"."user_fs";
+
+-- Could not auto-generate a down migration.
+-- Please write an appropriate down migration for the SQL below:
+-- DROP table "public"."transaction_comments";
diff --git a/hasura/migrations/default/1732201209408_squashed/up.sql b/hasura/migrations/default/1732201209408_squashed/up.sql
new file mode 100644
index 0000000..5585264
--- /dev/null
+++ b/hasura/migrations/default/1732201209408_squashed/up.sql
@@ -0,0 +1,12 @@
+
+DROP table "public"."transaction_comments";
+
+CREATE TABLE "public"."user_fs" ("pubkey" text NOT NULL, "index_request_cid" text NOT NULL, "data_cid" text NOT NULL, "time" timestamp NOT NULL, PRIMARY KEY ("pubkey") , UNIQUE ("pubkey"), UNIQUE ("index_request_cid"));COMMENT ON TABLE "public"."user_fs" IS E'User filesystem root cid';
+
+comment on column "public"."user_fs"."pubkey" is E'ss58 address of owner';
+
+comment on column "public"."user_fs"."index_request_cid" is E'CID of the latest index request that modified this document';
+
+comment on column "public"."user_fs"."data_cid" is E'CID of the user filesystem root';
+
+comment on column "public"."user_fs"."time" is E'timestamp of the latest index request that modified this document';
diff --git a/src/consts.ts b/src/consts.ts
index c45fe7b..986fdf4 100644
--- a/src/consts.ts
+++ b/src/consts.ts
@@ -15,12 +15,19 @@ export const EMPTY_NODE_CID = CID.parse('bafyreic3z6fetku5dpudahsne3dwbmwkddfnqx
// document kind of old cesium plus profile imported in the indexer
export const CESIUM_PLUS_PROFILE_IMPORT = 'cplus_raw'
-export const CESIUM_PLUS_PROFILE_INSERT = 'bafkreigi5phtqpo6a2f3tx4obaja4fzevy3nyvnl4bnkcxylyqnfeowzbm'
-export const CESIUM_PLUS_PROFILE_DELETE = 'bafkreic5bv5ytl7zv5rh5j2bd5mw6nfrn33mxhiobgmpsiu65yjw3eeduu'
-
-// document kind for transaction comment (old ones and new ones)
-export const TRANSACTION_COMMENT_V1 = 'TODO'
-export const TRANSACTION_COMMENT = 'bafkreiegjt5mrfj2hshuw6koejdfiykq57mzjeprfckxj5zpxxtqj4qzeu'
+export const CESIUM_PLUS_PROFILE_INSERT = 'cplus_upsert'
+export const CESIUM_PLUS_PROFILE_INSERT_OLD = 'bafkreigi5phtqpo6a2f3tx4obaja4fzevy3nyvnl4bnkcxylyqnfeowzbm' // deprecated
+export const CESIUM_PLUS_PROFILE_DELETE = 'cplus_delete'
+export const CESIUM_PLUS_PROFILE_DELETE_OLD = 'bafkreic5bv5ytl7zv5rh5j2bd5mw6nfrn33mxhiobgmpsiu65yjw3eeduu' // deprecated
+
+// document kind for transaction comment
+// tx comment index request is just a request to add to the tamt, not to the database
+export const TRANSACTION_COMMENT = 'dd_tx_comment'
+// other kind of tx comments are not decided yet
+export const TRANSACTION_COMMENT_OLD = 'bafkreiegjt5mrfj2hshuw6koejdfiykq57mzjeprfckxj5zpxxtqj4qzeu' // deprecated
+
+// index request for user public filesystem root hash
+export const DD_USER_FS = 'dd_user_fs'
// ==========
diff --git a/src/indexer/database.ts b/src/indexer/database.ts
index d0825cf..add13bd 100644
--- a/src/indexer/database.ts
+++ b/src/indexer/database.ts
@@ -2,7 +2,11 @@ import {
CESIUM_PLUS_PROFILE_IMPORT,
CESIUM_PLUS_PROFILE_INSERT,
CESIUM_PLUS_PROFILE_DELETE,
- TRANSACTION_COMMENT
+ TRANSACTION_COMMENT,
+ CESIUM_PLUS_PROFILE_DELETE_OLD,
+ CESIUM_PLUS_PROFILE_INSERT_OLD,
+ TRANSACTION_COMMENT_OLD,
+ DD_USER_FS
} from '../consts'
import type { CplusProfile, IndexRequest, TxComment } from '../types'
import { CID } from 'multiformats'
@@ -70,6 +74,34 @@ export async function setLatestIndexedCID(cid: CID) {
await setKey('last_indexed_cid', cid.toString())
}
+// query builder for user fs
+const userFsQb: QueryBuilder = {
+ query: `INSERT INTO
+ user_fs(index_request_cid, time, pubkey, data_cid)
+ VALUES ($1, $2, $3, $4,)
+ ON CONFLICT (pubkey)
+ DO UPDATE SET
+ index_request_cid = EXCLUDED.index_request_cid,
+ time = EXCLUDED.time,
+ pubkey = EXCLUDED.pubkey,
+ data_cid = EXCLUDED.data_cid,
+ WHERE EXCLUDED.time > profiles.time;
+ `,
+ // we do not need to fetch any data
+ dataGetter: (_dataCID) => Promise.resolve(null),
+ dataTransform: defaultDataTransform,
+ paramBuilder: (irCID: CID, ir: IndexRequest, dataCID: CID, _data: null) => [
+ // $1 index_request_cid
+ irCID.toString(),
+ // $2 time
+ new Date(ir.time).toISOString(),
+ // $3 pubkey
+ ir.pubkey,
+ // $4 data_cid
+ dataCID.toString(),
+ ]
+}
+
// cesium plus profile query and param builder
// completely overwrites previous data if new timestamp is higher
const cesiumPlusProfile: QueryBuilder = {
@@ -208,6 +240,7 @@ export async function handleInsertRequest(irCID: CID, ir: IndexRequest): Promise
switch (ir.kind) {
// insert cesium plus profile
case CESIUM_PLUS_PROFILE_INSERT:
+ case CESIUM_PLUS_PROFILE_INSERT_OLD:
return handleIrWithNonNullData<CplusProfile>(irCID, ir, cesiumPlusProfile)
// insert cesium plus import
@@ -218,12 +251,13 @@ export async function handleInsertRequest(irCID: CID, ir: IndexRequest): Promise
// delete cesium plus profile
case CESIUM_PLUS_PROFILE_DELETE:
+ case CESIUM_PLUS_PROFILE_DELETE_OLD:
// NOTE: if delete instruction is received from past, an existing profile can be deleted
// Cases when it can occur:
// - a profile is deleted, then re-created, then an attacker submits again the deletion message within the MAX_IR_TIME_DIFF delay
// - a node is synchronizing, gets profile creation from pubsub, and then gets old deletion request from peer
// NOTE: the same can happen if create instruction if received from the past
- // We could prevent it by:
+ // We could prevent it by:
// - keeping track of deleted profiles with a timestamp
// Not implemented yet because these cases have low probability, but this should be implemented in the future.
await client.query(`DELETE FROM profiles WHERE pubkey = $1;`, [ir.pubkey])
@@ -231,7 +265,15 @@ export async function handleInsertRequest(irCID: CID, ir: IndexRequest): Promise
// insert transaction comment
case TRANSACTION_COMMENT:
- return handleIrWithNonNullData<TxComment>(irCID, ir, txComment)
+ case TRANSACTION_COMMENT_OLD:
+ // we ignore offchain transaction comments in database for now since they happen on chain and are referenced by squid
+ // in the future we could be interested to index in clear offchain tx comments value to make them searchable
+ break
+
+ // user filesystem
+ case DD_USER_FS:
+ // the index request contains the data cid (non null) which is what we want to index (data is a fs that we do not look at)
+ return handleIrWithNonNullData<null>(irCID, ir, userFsQb)
// unimplemented
default:
--
GitLab