diff --git a/src/scripts/cesium-plus-import.ts b/src/scripts/cesium-plus-import.ts index c92eafc78f1cad9615e398d9a832ed9a49d78650..4a4d4e699d6557f69d8c9a22f3aae36abcc3cac6 100644 --- a/src/scripts/cesium-plus-import.ts +++ b/src/scripts/cesium-plus-import.ts @@ -9,6 +9,7 @@ import { import { createInterface } from 'readline' import { appendFile, createReadStream } from 'fs' import { timestampToKey } from '../processor' +import { readFile } from 'fs/promises' // profile files // const PROFILES = '/home/hugo/ipfs/v2s-datapod/migrate_csplus/profile_csplus.json' @@ -24,7 +25,7 @@ async function doImport() { const cids: CID[] = [] // manage chunk by chunk to limit memory usage for (let chunk = 0; chunk < CHUNKS; chunk++) { - const result = await fs.readFile(profiles(chunk), 'utf-8') + const result = await readFile(profiles(chunk), 'utf8') const obj = JSON.parse(result) console.log('chunk ' + chunk) await Promise.all(obj.map(processCesiumPlusProfile)).then((r: CID[]) => cids.push(...r)) @@ -144,8 +145,9 @@ async function putIrToIPFS() { console.log(`processed ${read} lines`) } }) - ).catch(e => { - console.log(e); + ) + .catch((e) => { + console.log(e) appendFile(rejected, line, () => { read++ }) @@ -160,8 +162,19 @@ async function putIrToIPFS() { } async function importAllCplusToAMT() { + const input = './input/cplusIR.txt' + const requests = await readFile(input, 'utf8') + .then((r) => r.split('\n')) + .then((p) => + p + .filter((e) => e.length > 0) + .map((e) => { + const parts = e.split(' ') + return [parts[0], CID.parse(parts[1])] as [string, CID] + }) + ) + .then((l) => l.sort()) const rootNodeCid = CID.parse('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid - const requests = await getAllCplusIr() await cplusIrToAMT(requests, rootNodeCid) } @@ -170,4 +183,5 @@ async function importAllCplusToAMT() { // doAllCplusCidsToAMT() // importAllCplusToAMT() // getAllCplusIr() -putIrToIPFS() +// putIrToIPFS() +// importAllCplusToAMT()