Select Git revision
cesium-plus.ts
-
Hugo Trentesaux authoredHugo Trentesaux authored
cesium-plus.ts 4.35 KiB
import { CID } from 'multiformats'
import { kubo } from './kubo'
import { Buffer } from 'buffer'
import { timestampToKey, arrayToVinode, mergeInodesSyncCID } from './processor'
import { CESIUM_PLUS_PROFILE_IMPORT } from './consts'
import type { CplusProfile, Avatar, IndexRequest } from './types'
// ========================= import functions
/// C+ profile to index request
function cplusProfileToIndexRequest(profile: CplusProfile, profileCid: CID): IndexRequest {
return {
pubkey: profile.issuer,
kind: CESIUM_PLUS_PROFILE_IMPORT,
data: profileCid,
time: profile.time * 1000,
sig: '' // signature is inside document for old C+ data
}
}
/// adds all cids by groups ot size `groupBy`
export async function processCesiumPlusImport(profileCids: CID[], groupBy: number): Promise<CID> {
const rootNode: Array<Promise<CID>> = []
for (let i = 0; i < Math.floor(profileCids.length / groupBy); i++) {
const group = profileCids.slice(groupBy * i, groupBy * (i + 1))
const cid = kubo.dag.put(group) as Promise<CID>
rootNode.push(cid)
}
return Promise.all(rootNode).then((r: CID[]) => kubo.dag.put(r) as Promise<CID>)
}
/// if avatar is present, upload it as a separate file instead
export async function processCesiumPlusProfile(obj: CplusProfile): Promise<CID> {
const { avatar, ...profileWithoutAvatar } = obj
if (avatar != undefined && (avatar as Avatar)._content != undefined) {
const buffer = Buffer.from((avatar as Avatar)._content, 'base64')
const fileCandidate = { content: new Uint8Array(buffer) }
return kubo
.add(fileCandidate)
.then((result) => kubo.dag.put({ ...profileWithoutAvatar, avatar: result.cid as CID }) as Promise<CID>)
} else {
return kubo.dag.put(obj) as Promise<CID>
}
}
// ========================= merge functions
/// retreive all C+ data as index requests sorted by key (timestamp)
async function allCplusAsIndexRequestCids(cplusrootCID: CID): Promise<Array<[string, CID]>> {
console.log(Date.now() + ' getting all cplus data')
const allCIDs: Array<Promise<[string, CID]>> = []
const cplusroot = await kubo.dag.get(cplusrootCID)
for (let chunkcid of cplusroot.value) {
const chunk = await kubo.dag.get(chunkcid)
for (let pcid of chunk.value) {
const profileIR: Promise<[string, CID]> = kubo.dag
.get(pcid)
.then((v) => cplusProfileToIndexRequest(v.value, pcid))
.then((r: IndexRequest) => Promise.all([timestampToKey(r.time), kubo.dag.put(r)] as [string, Promise<CID>]))
allCIDs.push(profileIR)
}
}
return Promise.all(allCIDs).then((r) => r.sort())
}
/// import cplus index requests to AMT
// about 90 seconds to get C+ data and convert to index requests
// about 90 seconds to merge data 1000 by 1000
// this makes less than 3 minutes in total
// requires maxSockets to be quite high
export async function cplusIndexRequestsToAMT(cplusrootCID: CID, rootNodeCid: CID) {
const chunkSize = 5000
console.log('getting all cplus index requests')
const requests = await allCplusAsIndexRequestCids(cplusrootCID)
const n = requests.length
console.log(Date.now() + ' merging')
for (let i = 0; i < n / chunkSize; i++) {
console.log(Date.now() + ' chunk number ' + i)
const chunk = requests.slice(i * chunkSize, (i + 1) * chunkSize)
const tree = arrayToVinode(chunk) // partial tree for this chunk
rootNodeCid = await mergeInodesSyncCID(rootNodeCid, tree)
console.log('new root node ' + rootNodeCid.toString())
}
}
/// wrap cplus raw document in index request
export async function cplusRawIrCID(cplus: CplusProfile, cplusRaw: string): Promise<CID> {
return kubo
.add(cplusRaw)
.then((cid) => ({
pubkey: cplus.issuer,
kind: CESIUM_PLUS_PROFILE_IMPORT,
data: cid,
time: cplus.time * 1000,
sig: null
}))
.then((ir) => kubo.dag.put(ir))
}
/// import cplus index requests to AMT
export async function cplusIrToAMT(requests: Array<[string, CID]>, rootNodeCid: CID) {
const chunkSize = 5000
const n = requests.length
console.log(Date.now() + ' merging')
for (let i = 0; i < n / chunkSize; i++) {
console.log(Date.now() + ' chunk number ' + i)
const chunk = requests.slice(i * chunkSize, (i + 1) * chunkSize)
const tree = arrayToVinode(chunk) // partial tree for this chunk
rootNodeCid = await mergeInodesSyncCID(rootNodeCid, tree)
console.log('new root node ' + rootNodeCid.toString())
}
}