index all data from amt

README.md

@@ -29,7 +29,7 @@ This will make easier to insert this data in any AMT or other data structure.
 # doAllCplusCidsToAMT()
 # takes about 180 seconds
 time npx tsx src/scripts/cesium-plus-import.ts
-# bafyreigczogsiuhaqus7eucalkwsy4vfkh3f4zg3c3rkvltxrwji6p5rnq
+# bafyreiffn3kkrakf7qj5m3wepsoxjwcojmesfyoexsvsovx23nhbfqu6bq
 ```
 
 ## Start collector and indexer

src/cesium-plus.ts

@@ -17,7 +17,8 @@ export interface CplusProfile {
   issuer: string
   hash: string
   signature: string
-  geoloc?: Geoloc
+  city?: string
+  geoPoint?: Geoloc
   socials?: Social[]
   tags?: string[]
   avatar?: Avatar | CID
@@ -37,8 +38,8 @@ interface Avatar {
 
 // geoloc
 interface Geoloc {
-  latitude: number
-  longitude: number
+  lat: number
+  lon: number
 }
 
 // ========================= import functions

src/indexer/database.ts

-import { CESIUM_PLUS_PROFILE_INSERT } from '../consts'
+import { CESIUM_PLUS_PROFILE_IMPORT, CESIUM_PLUS_PROFILE_INSERT } from '../consts'
 import type { IndexRequest } from '../types'
 import { CID } from 'multiformats'
 import pg from 'pg'
 import { kubo } from '../kubo'
+import type { CplusProfile } from '../cesium-plus'
 
 // TODO proper env
 const env = {
@@ -35,7 +36,10 @@ export async function getLatestIndexedCID(): Promise<CID | null> {
   }
 }
 export async function setLatestIndexedCID(cid: CID) {
-  // TODO
+  await client.query(
+    'INSERT INTO meta(key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = meta.value',
+    ['last_indexed_cid', cid.toString()]
+  )
 }
 
 // cesium plus profile query and param builder
@@ -52,7 +56,7 @@ const cesiumPlusProfile: QueryBuilder = {
         city = COALESCE(EXCLUDED.city, profiles.city),
         socials = COALESCE(EXCLUDED.socials, profiles.socials);
         `,
-  paramBuilder: (irCID: CID, ir: IndexRequest, dataCID: CID, data: any) => [
+  paramBuilder: (irCID: CID, ir: IndexRequest, dataCID: CID, data: CplusProfile) => [
     // $1 index_request_cid
     irCID.toString(),
     // $2 time
@@ -66,26 +70,50 @@ const cesiumPlusProfile: QueryBuilder = {
     // $6 description
     data.description,
     // $7 avatar
-    data.avatar.toString(),
+    data.avatar?.toString(),
     // $8 geoloc
-    data.geoloc?.latitude,
+    data.geoPoint?.lat,
     // $9
-    data.geoloc?.longitude,
+    data.geoPoint?.lon,
     // $10 city
     data.city,
     // $11 socials
-    data.socials
+    data.socials ? JSON.stringify(data.socials) : undefined
   ]
 }
 
 // insert index request in database
 export async function handleInsertRequest(irCID: CID, ir: IndexRequest) {
-  if (ir.kind.toString() == CESIUM_PLUS_PROFILE_INSERT.toString()) {
+  console.log('indexing ' + irCID)
+
+  switch (ir.kind.toString()) {
+    // insert new format
+    case CESIUM_PLUS_PROFILE_INSERT.toString(): {
+      const dataCID = ir.data
+      if (dataCID == null) {
+        console.error('no data')
+        return
+      }
+      const data = (await kubo.dag.get(dataCID)).value
+      await client.query(cesiumPlusProfile.query, cesiumPlusProfile.paramBuilder(irCID, ir, dataCID, data))
+      break
+    }
+
+    // insert old import
+    case CESIUM_PLUS_PROFILE_IMPORT.toString(): {
       const dataCID = ir.data
       if (dataCID == null) {
-      return // TODO log error
+        console.error('no data')
+        return
       }
       const data = (await kubo.dag.get(dataCID)).value
       await client.query(cesiumPlusProfile.query, cesiumPlusProfile.paramBuilder(irCID, ir, dataCID, data))
+      break
+    }
+
+    // unimplemented
+    default:
+      console.log('unimplemented kind ' + ir.kind)
+      break
   }
 }

src/indexer/handlers.ts

@@ -31,6 +31,9 @@ export async function indexStart(cid: CID) {
   isIndexing = true
   // read latest indexed root CID
   const latestCID = await getLatestIndexedCID()
+  console.log('latest indexed cid ' + latestCID)
+
+  if (latestCID?.toString() != cid.toString()) {
     // if defined iterate over all index requests of diff
     // else iterate over all index requests of given cid
     const iterator = latestCID ? getDiff(latestCID, cid) : getAll(cid)
@@ -39,6 +42,10 @@ export async function indexStart(cid: CID) {
     }
     // write root CID as lastest indexed CID
     await setLatestIndexedCID(cid)
+    console.log('new latest indexed cid ' + cid)
+  } else {
+    console.log('already indexed ' + cid)
+  }
   // unlock
   isIndexing = false
   // send check queue event to see if diff came in the meanwile

src/scripts/index-database.ts

@@ -5,16 +5,15 @@ import { getAll } from '../interface'
 import { kubo } from '../kubo'
 import type { IndexRequest } from '../types'
 
-const cid = CID.parse('bafyreiergrf2jnnh4tg4losvncm6jjgql4rsveadpxutvz2xa6zdb7azri')
+const cid = CID.parse('bafyreiffn3kkrakf7qj5m3wepsoxjwcojmesfyoexsvsovx23nhbfqu6bq')
+await indexStart(cid)
+// await client.end()
 
-// await indexStart(cid)
 
-
-const irCID = CID.parse('bafyreifpzcrghku22vp5oqzqgj4v3hxfhewe5d23kqdkhnkfapnrjgdoje')
-const ir = await kubo.dag.get(irCID)
-
-await handleInsertRequest(irCID, ir.value)
-await client.end()
+// // example to insert a specific request
+// const irCID = CID.parse('bafyreifpzcrghku22vp5oqzqgj4v3hxfhewe5d23kqdkhnkfapnrjgdoje')
+// const ir = await kubo.dag.get(irCID)
+// await handleInsertRequest(irCID, ir.value)
 
 // // example to iterate over all
 // for await (const leaf of getAll(cid)) {