Skip to content
Snippets Groups Projects
Commit e441b7dc authored by Hugo Trentesaux's avatar Hugo Trentesaux
Browse files

import raw cplus to amt

parent 6dc4598f
No related branches found
No related tags found
No related merge requests found
......@@ -9,6 +9,7 @@ import {
import { createInterface } from 'readline'
import { appendFile, createReadStream } from 'fs'
import { timestampToKey } from '../processor'
import { readFile } from 'fs/promises'
// profile files
// const PROFILES = '/home/hugo/ipfs/v2s-datapod/migrate_csplus/profile_csplus.json'
......@@ -24,7 +25,7 @@ async function doImport() {
const cids: CID[] = []
// manage chunk by chunk to limit memory usage
for (let chunk = 0; chunk < CHUNKS; chunk++) {
const result = await fs.readFile(profiles(chunk), 'utf-8')
const result = await readFile(profiles(chunk), 'utf8')
const obj = JSON.parse(result)
console.log('chunk ' + chunk)
await Promise.all(obj.map(processCesiumPlusProfile)).then((r: CID[]) => cids.push(...r))
......@@ -144,8 +145,9 @@ async function putIrToIPFS() {
console.log(`processed ${read} lines`)
}
})
).catch(e => {
console.log(e);
)
.catch((e) => {
console.log(e)
appendFile(rejected, line, () => {
read++
})
......@@ -160,8 +162,19 @@ async function putIrToIPFS() {
}
async function importAllCplusToAMT() {
const input = './input/cplusIR.txt'
const requests = await readFile(input, 'utf8')
.then((r) => r.split('\n'))
.then((p) =>
p
.filter((e) => e.length > 0)
.map((e) => {
const parts = e.split(' ')
return [parts[0], CID.parse(parts[1])] as [string, CID]
})
)
.then((l) => l.sort())
const rootNodeCid = CID.parse('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid
const requests = await getAllCplusIr()
await cplusIrToAMT(requests, rootNodeCid)
}
......@@ -170,4 +183,5 @@ async function importAllCplusToAMT() {
// doAllCplusCidsToAMT()
// importAllCplusToAMT()
// getAllCplusIr()
putIrToIPFS()
// putIrToIPFS()
// importAllCplusToAMT()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment