diff --git a/README.md b/README.md index ed71cb3ada4e1f71b737901cd33dbf3d916ae01e..32b2f96898d81a191313d2e260e211fb3202a10b 100644 --- a/README.md +++ b/README.md @@ -80,13 +80,17 @@ Features - [x] pubkey instead of ss58 address if we want data to be compatible across networks → ss58 - [ ] add periodic sync with a list of IPNS - [ ] split indexer vue app from backend indexer and improve node admin app - - [ ] clarify the purpose of the the main TAMT + - [x] clarify the purpose of the the main TAMT - [ ] clarify the adressing format in the tables - [ ] add domain specific index for profile for example - [ ] add a refcount to count the number of documents - - [ ] + - [ ] make the app build in prod mode + - [ ] allow connecting the app to a custom RPC endpoint - [ ] manage unpin requests when user/admin wants to delete data, see refcount - [ ] document dev database change with tracking hasura console and squashing migrations - [ ] add transaction comment -- [ ] add version history to database (history of index request CIDs) +- [ ] add version history to database (history of index request CIDs) -> not systematic - [ ] update description of pubkey field to ss58 +- [ ] add ability to remove a node as well as its parent if it leaves it empty +- [ ] make base custom per tree (base 16, base 32) +- [ ] diff --git a/src/indexer/start.ts b/src/indexer/start.ts index 26b972fc35c9bddbaca70cd64d6e920a70db7792..97ef7e23074367952fccbc831647b9a272a5a648 100644 --- a/src/indexer/start.ts +++ b/src/indexer/start.ts @@ -11,15 +11,28 @@ import { getRootCIDfromArgs } from './utils' // === HANDLERS === // pubsub message handler -async function validMessageHandler(cid: CID, dag: IndexRequest): Promise<void> { - console.log('adding valid index request to process queue') - // add index request to the process list - processQueue.push([cid, dag]) - // ask to process the request - events.emit(evtype.trigger) - // try pin the new data so that it is available to the indexer - // there might be a UND_ERR_BODY_TIMEOUT if data could not be retreived in time - kubo.pin.add(cid, { recursive: true }).catch(() => console.log('📌 could not pin valid index request ' + cid)) +async function validMessageHandler(_cid: CID, dag: IndexRequest): Promise<void> { + // re-build the index request (because the type is loosely defined) + const ir: IndexRequest = { + kind: dag.kind, + time: dag.time, + data: dag.data, + pubkey: dag.pubkey, + sig: dag.sig + } + // then store the index request locally + kubo.dag + .put(ir, { pin: true }) + .then((cid) => { + // cids should be the same + if (cid.toString() != _cid.toString()) console.log('⚠️ ' + cid + ' != ' + _cid) + console.log('adding valid index request to process queue') + // add index request to the process list + processQueue.push([cid, ir]) + // ask to process the request + events.emit(evtype.trigger) + }) + .catch(() => console.log('📌 could not add valid index request ' + _cid)) return } const handleMessage = getPubSubHandler(validMessageHandler) @@ -46,7 +59,7 @@ function handleBatch() { items.push(i) } // convert it to a list of [key, cid] for batch insert (merge) - const requests = items.map(([cid, dag]) => [timestampToKey(dag.time), cid]).sort() as Array<[string, CID]> + const requests = items.map(([cid, ir]) => [timestampToKey(ir.time), cid]).sort() as Array<[string, CID]> const tree = arrayToVinode(requests) // insert them