Skip to content
Snippets Groups Projects
Select Git revision
  • main default protected
1 result

cesium-plus.ts

Blame
  • cesium-plus.ts 4.35 KiB
    import { CID } from 'multiformats'
    import { kubo } from './kubo'
    import { Buffer } from 'buffer'
    import { timestampToKey, arrayToVinode, mergeInodesSyncCID } from './processor'
    import { CESIUM_PLUS_PROFILE_IMPORT } from './consts'
    import type { CplusProfile, Avatar, IndexRequest } from './types'
    
    // ========================= import functions
    
    /// C+ profile to index request
    function cplusProfileToIndexRequest(profile: CplusProfile, profileCid: CID): IndexRequest {
      return {
        pubkey: profile.issuer,
        kind: CESIUM_PLUS_PROFILE_IMPORT,
        data: profileCid,
        time: profile.time * 1000,
        sig: '' // signature is inside document for old C+ data
      }
    }
    
    /// adds all cids by groups ot size `groupBy`
    export async function processCesiumPlusImport(profileCids: CID[], groupBy: number): Promise<CID> {
      const rootNode: Array<Promise<CID>> = []
      for (let i = 0; i < Math.floor(profileCids.length / groupBy); i++) {
        const group = profileCids.slice(groupBy * i, groupBy * (i + 1))
        const cid = kubo.dag.put(group) as Promise<CID>
        rootNode.push(cid)
      }
      return Promise.all(rootNode).then((r: CID[]) => kubo.dag.put(r) as Promise<CID>)
    }
    
    /// if avatar is present, upload it as a separate file instead
    export async function processCesiumPlusProfile(obj: CplusProfile): Promise<CID> {
      const { avatar, ...profileWithoutAvatar } = obj
      if (avatar != undefined && (avatar as Avatar)._content != undefined) {
        const buffer = Buffer.from((avatar as Avatar)._content, 'base64')
        const fileCandidate = { content: new Uint8Array(buffer) }
        return kubo
          .add(fileCandidate)
          .then((result) => kubo.dag.put({ ...profileWithoutAvatar, avatar: result.cid as CID }) as Promise<CID>)
      } else {
        return kubo.dag.put(obj) as Promise<CID>
      }
    }
    
    // ========================= merge functions
    
    /// retreive all C+ data as index requests sorted by key (timestamp)
    async function allCplusAsIndexRequestCids(cplusrootCID: CID): Promise<Array<[string, CID]>> {
      console.log(Date.now() + ' getting all cplus data')
      const allCIDs: Array<Promise<[string, CID]>> = []
      const cplusroot = await kubo.dag.get(cplusrootCID)
      for (let chunkcid of cplusroot.value) {
        const chunk = await kubo.dag.get(chunkcid)
        for (let pcid of chunk.value) {
          const profileIR: Promise<[string, CID]> = kubo.dag
            .get(pcid)
            .then((v) => cplusProfileToIndexRequest(v.value, pcid))
            .then((r: IndexRequest) => Promise.all([timestampToKey(r.time), kubo.dag.put(r)] as [string, Promise<CID>]))
          allCIDs.push(profileIR)
        }
      }
      return Promise.all(allCIDs).then((r) => r.sort())
    }
    
    /// import cplus index requests to AMT
    // about 90 seconds to get C+ data and convert to index requests
    // about 90 seconds to merge data 1000 by 1000
    // this makes less than 3 minutes in total
    // requires maxSockets to be quite high
    export async function cplusIndexRequestsToAMT(cplusrootCID: CID, rootNodeCid: CID) {
      const chunkSize = 5000
      console.log('getting all cplus index requests')
      const requests = await allCplusAsIndexRequestCids(cplusrootCID)
      const n = requests.length
      console.log(Date.now() + ' merging')
      for (let i = 0; i < n / chunkSize; i++) {
        console.log(Date.now() + ' chunk number ' + i)
        const chunk = requests.slice(i * chunkSize, (i + 1) * chunkSize)
        const tree = arrayToVinode(chunk) // partial tree for this chunk
        rootNodeCid = await mergeInodesSyncCID(rootNodeCid, tree)
        console.log('new root node ' + rootNodeCid.toString())
      }
    }
    
    /// wrap cplus raw document in index request
    export async function cplusRawIrCID(cplus: CplusProfile, cplusRaw: string): Promise<CID> {
      return kubo
        .add(cplusRaw)
        .then((cid) => ({
          pubkey: cplus.issuer,
          kind: CESIUM_PLUS_PROFILE_IMPORT,
          data: cid,
          time: cplus.time * 1000,
          sig: null
        }))
        .then((ir) => kubo.dag.put(ir))
    }
    
    /// import cplus index requests to AMT
    export async function cplusIrToAMT(requests: Array<[string, CID]>, rootNodeCid: CID) {
      const chunkSize = 5000
      const n = requests.length
      console.log(Date.now() + ' merging')
      for (let i = 0; i < n / chunkSize; i++) {
        console.log(Date.now() + ' chunk number ' + i)
        const chunk = requests.slice(i * chunkSize, (i + 1) * chunkSize)
        const tree = arrayToVinode(chunk) // partial tree for this chunk
        rootNodeCid = await mergeInodesSyncCID(rootNodeCid, tree)
        console.log('new root node ' + rootNodeCid.toString())
      }
    }