cesium-plus-import.ts

import { CID } from 'multiformats'
import { processCesiumPlusImport, processCesiumPlusProfile, cplusIndexRequestsToAMT } from '../cesium-plus'
import * as fs from 'fs/promises'

// profile files
// const PROFILES = '/home/hugo/ipfs/v2s-datapod/migrate_csplus/profile_csplus.json'
const profiles = (n: number) => `/home/hugo/ipfs/v2s-datapod/migrate_csplus/profile_csplus_${n}.json`
const CHUNKS = 11
const GROUPBY = 256

/// do import cesium data from chunk files to a single basic root dag
// use json chunks as input
// process base64 images apart
// groups output in a single dag for easy import
async function doImport() {
  const cids: CID[] = []
  // manage chunk by chunk to limit memory usage
  for (let chunk = 0; chunk < CHUNKS; chunk++) {
    const result = await fs.readFile(profiles(chunk), 'utf-8')
    const obj = JSON.parse(result)
    console.log('chunk ' + chunk)
    await Promise.all(obj.map(processCesiumPlusProfile)).then((r: CID[]) => cids.push(...r))
  }
  console.log('processing...')
  console.log(cids.length)

  processCesiumPlusImport(cids, GROUPBY).then((cid) => console.log(cid))
}

// // in node, this is a bit faster than in chromium, and we get about 25 profiles per second instead of 18
// // but this shows we should optimise AMT inserting for higher throughput
// function doImportToAMT() {
// REMOVED
// }

// // this is a more optimized version that takes 50 seconds to import all 50000 profiles → 1000 profiles per second
// // this version sometimes crashes with EADDRNOTAVAIL because it exceeds the number of concurrent connections
// async function doMergeAMT() {
// REMOVED
// }

/// import all previously imported C+ data as index requests into a single AMT
async function doAllCplusCidsToAMT() {
  const cplusCID = CID.parse('bafyreie74jtf23zzz2tdgsz7axfrm4pidje43ypqn25v4gkdtfjbcj62km') // cesium plus import
  const rootNodeCid = CID.parse('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid

  cplusIndexRequestsToAMT(cplusCID, rootNodeCid)
}

// TODO use command line args to choose what to do
// doImport()
// doAllCplusCidsToAMT()