wip

README.md

@@ -80,13 +80,17 @@ Features
 - [x] pubkey instead of ss58 address if we want data to be compatible across networks → ss58
 - [ ] add periodic sync with a list of IPNS
 - [ ] split indexer vue app from backend indexer and improve node admin app
-    - [ ] clarify the purpose of the the main TAMT
+    - [x] clarify the purpose of the the main TAMT
     - [ ] clarify the adressing format in the tables
     - [ ] add domain specific index for profile for example
     - [ ] add a refcount to count the number of documents
-    - [ ] 
+    - [ ] make the app build in prod mode
+    - [ ] allow connecting the app to a custom RPC endpoint
 - [ ] manage unpin requests when user/admin wants to delete data, see refcount
 - [ ] document dev database change with tracking hasura console and squashing migrations
 - [ ] add transaction comment
-- [ ] add version history to database (history of index request CIDs)
+- [ ] add version history to database (history of index request CIDs) -> not systematic
 - [ ] update description of pubkey field to ss58
+- [ ] add ability to remove a node as well as its parent if it leaves it empty
+- [ ] make base custom per tree (base 16, base 32)
+- [ ] 

src/indexer/start.ts

@@ -11,15 +11,28 @@ import { getRootCIDfromArgs } from './utils'
 // === HANDLERS ===
 
 // pubsub message handler
-async function validMessageHandler(cid: CID, dag: IndexRequest): Promise<void> {
+async function validMessageHandler(_cid: CID, dag: IndexRequest): Promise<void> {
+  // re-build the index request (because the type is loosely defined)
+  const ir: IndexRequest = {
+    kind: dag.kind,
+    time: dag.time,
+    data: dag.data,
+    pubkey: dag.pubkey,
+    sig: dag.sig
+  }
+  // then store the index request locally
+  kubo.dag
+    .put(ir, { pin: true })
+    .then((cid) => {
+      // cids should be the same
+      if (cid.toString() != _cid.toString()) console.log('⚠️ ' + cid + ' != ' + _cid)
       console.log('adding valid index request to process queue')
       // add index request to the process list
-  processQueue.push([cid, dag])
+      processQueue.push([cid, ir])
       // ask to process the request
       events.emit(evtype.trigger)
-  // try pin the new data so that it is available to the indexer
-  // there might be a UND_ERR_BODY_TIMEOUT if data could not be retreived in time
-  kubo.pin.add(cid, { recursive: true }).catch(() => console.log('📌 could not pin valid index request ' + cid))
+    })
+    .catch(() => console.log('📌 could not add valid index request ' + _cid))
   return
 }
 const handleMessage = getPubSubHandler(validMessageHandler)
@@ -46,7 +59,7 @@ function handleBatch() {
     items.push(i)
   }
   // convert it to a list of [key, cid] for batch insert (merge)
-  const requests = items.map(([cid, dag]) => [timestampToKey(dag.time), cid]).sort() as Array<[string, CID]>
+  const requests = items.map(([cid, ir]) => [timestampToKey(ir.time), cid]).sort() as Array<[string, CID]>
   const tree = arrayToVinode(requests)
 
   // insert them