diff --git a/README.md b/README.md index 72e96076c9a2fa2d80daf63c024974257e18412a..df9144d3c1cd5f02114daea47ddf46c349f8aa28 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ This will make easier to insert this data in any AMT or other data structure. # doAllCplusCidsToAMT() # takes about 180 seconds time npx tsx src/scripts/cesium-plus-import.ts -# bafyreigczogsiuhaqus7eucalkwsy4vfkh3f4zg3c3rkvltxrwji6p5rnq +# bafyreiffn3kkrakf7qj5m3wepsoxjwcojmesfyoexsvsovx23nhbfqu6bq ``` ## Start collector and indexer diff --git a/src/cesium-plus.ts b/src/cesium-plus.ts index b5a4fa589c588bc915ae738f5867dd5d84f12464..b428229fb7789e8d9b3b17e685fc4cb6d28f6d8c 100644 --- a/src/cesium-plus.ts +++ b/src/cesium-plus.ts @@ -17,7 +17,8 @@ export interface CplusProfile { issuer: string hash: string signature: string - geoloc?: Geoloc + city?: string + geoPoint?: Geoloc socials?: Social[] tags?: string[] avatar?: Avatar | CID @@ -37,8 +38,8 @@ interface Avatar { // geoloc interface Geoloc { - latitude: number - longitude: number + lat: number + lon: number } // ========================= import functions diff --git a/src/indexer/database.ts b/src/indexer/database.ts index d140243a613342c14f520bfe207e79e99b0b9920..951295a9ba5beac4dec3748bfca29e08d6276d3c 100644 --- a/src/indexer/database.ts +++ b/src/indexer/database.ts @@ -1,8 +1,9 @@ -import { CESIUM_PLUS_PROFILE_INSERT } from '../consts' +import { CESIUM_PLUS_PROFILE_IMPORT, CESIUM_PLUS_PROFILE_INSERT } from '../consts' import type { IndexRequest } from '../types' import { CID } from 'multiformats' import pg from 'pg' import { kubo } from '../kubo' +import type { CplusProfile } from '../cesium-plus' // TODO proper env const env = { @@ -35,7 +36,10 @@ export async function getLatestIndexedCID(): Promise<CID | null> { } } export async function setLatestIndexedCID(cid: CID) { - // TODO + await client.query( + 'INSERT INTO meta(key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = meta.value', + ['last_indexed_cid', cid.toString()] + ) } // cesium plus profile query and param builder @@ -52,7 +56,7 @@ const cesiumPlusProfile: QueryBuilder = { city = COALESCE(EXCLUDED.city, profiles.city), socials = COALESCE(EXCLUDED.socials, profiles.socials); `, - paramBuilder: (irCID: CID, ir: IndexRequest, dataCID: CID, data: any) => [ + paramBuilder: (irCID: CID, ir: IndexRequest, dataCID: CID, data: CplusProfile) => [ // $1 index_request_cid irCID.toString(), // $2 time @@ -66,26 +70,50 @@ const cesiumPlusProfile: QueryBuilder = { // $6 description data.description, // $7 avatar - data.avatar.toString(), + data.avatar?.toString(), // $8 geoloc - data.geoloc?.latitude, + data.geoPoint?.lat, // $9 - data.geoloc?.longitude, + data.geoPoint?.lon, // $10 city data.city, // $11 socials - data.socials + data.socials ? JSON.stringify(data.socials) : undefined ] } // insert index request in database export async function handleInsertRequest(irCID: CID, ir: IndexRequest) { - if (ir.kind.toString() == CESIUM_PLUS_PROFILE_INSERT.toString()) { - const dataCID = ir.data - if (dataCID == null) { - return // TODO log error + console.log('indexing ' + irCID) + + switch (ir.kind.toString()) { + // insert new format + case CESIUM_PLUS_PROFILE_INSERT.toString(): { + const dataCID = ir.data + if (dataCID == null) { + console.error('no data') + return + } + const data = (await kubo.dag.get(dataCID)).value + await client.query(cesiumPlusProfile.query, cesiumPlusProfile.paramBuilder(irCID, ir, dataCID, data)) + break + } + + // insert old import + case CESIUM_PLUS_PROFILE_IMPORT.toString(): { + const dataCID = ir.data + if (dataCID == null) { + console.error('no data') + return + } + const data = (await kubo.dag.get(dataCID)).value + await client.query(cesiumPlusProfile.query, cesiumPlusProfile.paramBuilder(irCID, ir, dataCID, data)) + break } - const data = (await kubo.dag.get(dataCID)).value - await client.query(cesiumPlusProfile.query, cesiumPlusProfile.paramBuilder(irCID, ir, dataCID, data)) + + // unimplemented + default: + console.log('unimplemented kind ' + ir.kind) + break } } diff --git a/src/indexer/handlers.ts b/src/indexer/handlers.ts index 3fd5a15272e469af60adefd422569e88bab6bf9a..7e5e18abe5f36d05ab87a59a016ff8e563f5cba1 100644 --- a/src/indexer/handlers.ts +++ b/src/indexer/handlers.ts @@ -31,14 +31,21 @@ export async function indexStart(cid: CID) { isIndexing = true // read latest indexed root CID const latestCID = await getLatestIndexedCID() - // if defined iterate over all index requests of diff - // else iterate over all index requests of given cid - const iterator = latestCID ? getDiff(latestCID, cid) : getAll(cid) - for await (const leaf of iterator) { - leaf.forEach((irCID) => kubo.dag.get(irCID).then((ir) => handleInsertRequest(irCID, ir.value))) + console.log('latest indexed cid ' + latestCID) + + if (latestCID?.toString() != cid.toString()) { + // if defined iterate over all index requests of diff + // else iterate over all index requests of given cid + const iterator = latestCID ? getDiff(latestCID, cid) : getAll(cid) + for await (const leaf of iterator) { + leaf.forEach((irCID) => kubo.dag.get(irCID).then((ir) => handleInsertRequest(irCID, ir.value))) + } + // write root CID as lastest indexed CID + await setLatestIndexedCID(cid) + console.log('new latest indexed cid ' + cid) + } else { + console.log('already indexed ' + cid) } - // write root CID as lastest indexed CID - await setLatestIndexedCID(cid) // unlock isIndexing = false // send check queue event to see if diff came in the meanwile diff --git a/src/scripts/index-database.ts b/src/scripts/index-database.ts index 0b9680676f77be52ddd36433b26ea7bf4e8badad..5e1bb17eff0b78ebba0541d962ad5e7805139885 100644 --- a/src/scripts/index-database.ts +++ b/src/scripts/index-database.ts @@ -5,16 +5,15 @@ import { getAll } from '../interface' import { kubo } from '../kubo' import type { IndexRequest } from '../types' -const cid = CID.parse('bafyreiergrf2jnnh4tg4losvncm6jjgql4rsveadpxutvz2xa6zdb7azri') +const cid = CID.parse('bafyreiffn3kkrakf7qj5m3wepsoxjwcojmesfyoexsvsovx23nhbfqu6bq') +await indexStart(cid) +// await client.end() -// await indexStart(cid) - -const irCID = CID.parse('bafyreifpzcrghku22vp5oqzqgj4v3hxfhewe5d23kqdkhnkfapnrjgdoje') -const ir = await kubo.dag.get(irCID) - -await handleInsertRequest(irCID, ir.value) -await client.end() +// // example to insert a specific request +// const irCID = CID.parse('bafyreifpzcrghku22vp5oqzqgj4v3hxfhewe5d23kqdkhnkfapnrjgdoje') +// const ir = await kubo.dag.get(irCID) +// await handleInsertRequest(irCID, ir.value) // // example to iterate over all // for await (const leaf of getAll(cid)) {