diff --git a/src/cesium-plus.ts b/src/cesium-plus.ts index 5ca1a1cd9b48509b3a1fb603a44e460897aa9b38..6f250354b34f181859f74c9eb2eb872b0cd4ae63 100644 --- a/src/cesium-plus.ts +++ b/src/cesium-plus.ts @@ -1,6 +1,8 @@ import { CID } from 'multiformats' import { kubo } from './kubo' import { Buffer } from 'buffer' +import { insertKnownIndexRequest } from './processor' +import type { IndexRequest, Pointer } from './types' // for reference see // https://doc.e-is.pro/cesium-plus-pod/REST_API.html @@ -55,3 +57,52 @@ export async function processCesiumPlusProfile(obj: CplusProfile): Promise<CID> return kubo.dag.put(profile) as Promise<CID> } } + +// import these cid to target AMT +export async function importCplusToAMT(cplusCID: CID, amtCid: Pointer<CID>) { + const cplusroot = await kubo.dag.get(cplusCID) + for (let chunkcid of cplusroot.value) { + // process each chunk sequentially to avoid memory overflow + const chunk = await kubo.dag.get(chunkcid) + chunk.value.map((pcid: CID) => + kubo.dag.get(pcid).then((p) => { + const profile: CplusProfile = p.value + const indexRequest: IndexRequest = { + pubkey: profile.issuer, + cid: pcid, + timestamp: profile.time, + signature: '' // signature is inside document for C+ data + } + kubo.dag.put(indexRequest).then(async (indexRequestCid) => { + // update CID synchronously for next insert + amtCid.value = await insertKnownIndexRequest(amtCid.value, indexRequestCid as CID, indexRequest) + }) + }) + ) + } +} + +// alternative slow synchronous function to avoid overflowing kubo with connections +// exectued in chromium, it can import about 18 profiles per second +export async function importCplusToAMTSync(cplusCID: CID, amtCid: CID): Promise<CID> { + const cplusroot = await kubo.dag.get(cplusCID) + let importedCount = 0 + for (let chunkcid of cplusroot.value) { + const chunk = await kubo.dag.get(chunkcid) + for (let pcid of chunk.value) { + const p = await kubo.dag.get(pcid) + const profile: CplusProfile = p.value + const indexRequest: IndexRequest = { + pubkey: profile.issuer, + cid: pcid, + timestamp: profile.time, + signature: '' // signature is inside document for C+ data + } + const indexRequestCid = await kubo.dag.put(indexRequest) + amtCid = await insertKnownIndexRequest(amtCid, indexRequestCid as CID, indexRequest) + importedCount += 1 + console.log(importedCount) + } + } + return amtCid +} diff --git a/src/processor.ts b/src/processor.ts index 9a51b3227147e72dc049e0f04929d49043dd8cd2..5eee07a798e422e480922bfc1b37d2a3574fdde2 100644 --- a/src/processor.ts +++ b/src/processor.ts @@ -30,11 +30,11 @@ export async function addToIndexQueue(cid: CID, indexRequest: IndexRequest) { export async function insertIndexRequest(rootCid: CID, indexRequestCid: CID): Promise<CID> { const [rootDag, indexDag] = await Promise.all([kubo.dag.get(rootCid), kubo.dag.get(indexRequestCid)]) const key = indexDag.value.timestamp.toString(BASE).padStart(KEYSIZE, '0') - return processInode(rootDag.value, key, indexDag.value) + return processInode(rootDag.value, key, indexRequestCid) } // same as above but with known request -async function insertKnownIndexRequest(rootCid: CID, indexRequestCid: CID, indexRequest: IndexRequest): Promise<CID> { +export async function insertKnownIndexRequest(rootCid: CID, indexRequestCid: CID, indexRequest: IndexRequest): Promise<CID> { const key = indexRequest.timestamp.toString(BASE).padStart(KEYSIZE, '0') const rootDag = await kubo.dag.get(rootCid) return processInode(rootDag.value, key, indexRequestCid) diff --git a/src/scripts/cesium-plus-import.ts b/src/scripts/cesium-plus-import.ts index 295abd302cb3f53064392ecab2f9d4007ead2ffb..f3b3067be53405497fee3e286f3e8b12fdc79885 100644 --- a/src/scripts/cesium-plus-import.ts +++ b/src/scripts/cesium-plus-import.ts @@ -1,5 +1,5 @@ -import type { CID } from 'multiformats' -import { processCesiumPlusImport, processCesiumPlusProfile } from '../cesium-plus' +import { CID } from 'multiformats' +import { processCesiumPlusImport, processCesiumPlusProfile, importCplusToAMTSync } from '../cesium-plus' import * as fs from 'fs/promises' // profile files @@ -23,4 +23,14 @@ async function doImport() { processCesiumPlusImport(cids, GROUPBY).then((cid) => console.log(cid)) } -doImport() +// doImport() + +// in node, this is a bit faster than in chromium, and we get about 25 profiles per second instead of 18 +// but this shows we should optimise AMT inserting for higher throughput +function doImportToAMT() { + const cplus = CID.parse('bafyreie74jtf23zzz2tdgsz7axfrm4pidje43ypqn25v4gkdtfjbcj62km') // cesium plus import + const amt = CID.parse('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid + importCplusToAMTSync(cplus, amt).then(console.log) +} + +doImportToAMT() \ No newline at end of file diff --git a/src/types.ts b/src/types.ts index 09c103a8aa3e04ac84fa49e883e42e59a726df74..0cfd81b01b56faf6ce8c0f9401538de697668065 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,6 +1,10 @@ import { CID } from 'multiformats' import { BASE } from './consts' +export interface Pointer<T> { + value: T +} + export interface IndexRequest { pubkey: string cid: CID diff --git a/src/views/CplusView.vue b/src/views/CplusView.vue index e4cd5ba7060e2a0bf9aaefe2a3a31d022ea1a462..f49a4daf55d3a0bf4366a827f20e372a48f5b8ab 100644 --- a/src/views/CplusView.vue +++ b/src/views/CplusView.vue @@ -1,16 +1,33 @@ <script setup lang="ts"> -import { ref, type Ref, computed } from 'vue' -import { processCesiumPlusProfile, type CplusProfile } from '../cesium-plus' +import { ref, type Ref, onMounted } from 'vue' +import { processCesiumPlusProfile, type CplusProfile, importCplusToAMTSync } from '../cesium-plus' +import { CID } from 'multiformats' const file: Ref<File | null> = ref(null) const sample = ref('') const cidlist = ref('') const MAX = 20 - +const leftMsg = ref('bafyreie74jtf23zzz2tdgsz7axfrm4pidje43ypqn25v4gkdtfjbcj62km') // cesium plus import +const rightMsg = ref('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid +const leftRootCid: Ref<CID | null> = ref(null) +const rightRootCid: Ref<CID | null> = ref(null) +function setLeftRoot() { + leftRootCid.value = CID.parse(leftMsg.value) +} +function setRightRoot() { + rightRootCid.value = CID.parse(rightMsg.value) +} function setfile(input: any) { file.value = input.target.files[0] } +onMounted(() => { + try { + setLeftRoot() + setRightRoot() + } catch {} +}) + function importCplus() { cidlist.value = '' if (file.value) { @@ -27,6 +44,12 @@ function importCplus() { reader.readAsText(file.value) } } + +function doImportCplusToAMT() { + if (leftRootCid.value != null && rightRootCid.value != null) { + importCplusToAMTSync(leftRootCid.value, rightRootCid.value).then((c) => (rightRootCid.value = c)) + } +} </script> <template> @@ -47,6 +70,19 @@ function importCplus() { <h2>Import from root cid</h2> <p>Choose a root CID from Cesium import and the root CID of the target AMT</p> + <p> + <input v-model="leftMsg" @keyup.enter="setLeftRoot" /> + <button v-on:click="setLeftRoot">set cesium root</button> + <input v-model="rightMsg" @keyup.enter="setRightRoot" /> + <button v-on:click="setRightRoot">set target AMT</button> + </p> + <p class="mono">{{ leftRootCid }} {{ rightRootCid }}</p> + <p> + <button v-on:click="doImportCplusToAMT">import !</button> + </p> + <p> + {{ rightRootCid?.toString() }} + </p> </template> <style scoped></style> diff --git a/src/views/DiffView.vue b/src/views/DiffView.vue index 0ca51a1f704e29dc6f1f61b9b1cdbfa961dd000f..0b7809e7dd8b80d5eb4fd0d436deba1dfaeaebf3 100644 --- a/src/views/DiffView.vue +++ b/src/views/DiffView.vue @@ -3,7 +3,7 @@ import { kubo } from '@/kubo' import { emptyInode, type IndexHist, type IndexInode, type IndexLeaf } from '../types' import { IPNS_HIST, BASE, KEYSIZE } from '../consts' import { CID } from 'multiformats' -import { ref, type Ref, computed } from 'vue' +import { ref, type Ref, onMounted } from 'vue' const leftMsg = ref('') const rightMsg = ref('') @@ -179,14 +179,16 @@ function initWithLastCommit() { resolveHist().then((histCid) => { kubo.dag.get(histCid).then((hist) => { rightRootCid.value = hist.value.current_index - kubo.dag.get(hist.value.last_history).then((hist1) => { - leftRootCid.value = hist1?.value.current_index - }) + if (hist.value.last_history) { + kubo.dag.get(hist.value.last_history).then((hist1) => { + leftRootCid.value = hist1?.value.current_index + }) + } }) }) } -initWithLastCommit() +onMounted(initWithLastCommit) </script> <template> diff --git a/src/views/IndexView.vue b/src/views/IndexView.vue index cdb9a468b76fe5b859fac930ba0df28188a2ded1..fdfdd5229730af86efc08dc944fe8b7d92bc2c38 100644 --- a/src/views/IndexView.vue +++ b/src/views/IndexView.vue @@ -3,7 +3,7 @@ import { kubo } from '@/kubo' import { emptyInode, type IndexHist } from '../types' import { IPNS, IPNS_HIST } from '../consts' import { CID } from 'multiformats' -import { ref, type Ref, computed } from 'vue' +import { ref, type Ref, computed, onMounted } from 'vue' import IndexNode from '../components/IndexNode.vue' import IndexHistory from '../components/IndexHistory.vue' @@ -35,7 +35,7 @@ async function initIndex() { // resolve again to confirm publish result resolve() // update history accordingly - updateHist(rootcid) + updateHist(rootcid as CID) } // inits indexing and publishes ipns entry for it @@ -54,7 +54,6 @@ async function initIndexHist() { async function resolve() { for await (const name of kubo.name.resolve(IPNS, { nocache: true })) { - console.log(name) ipnsTarget.value = name targetCid.value = CID.parse(name.slice(6)) } @@ -91,8 +90,10 @@ function setTargetCid() { } } -resolve() -resolveHist() +onMounted(() => { + resolve() + resolveHist() +}) </script> <template> diff --git a/src/views/KuboView.vue b/src/views/KuboView.vue index b2969abb9c04b892892867ee0bcfd09ea5b3d2bd..dd12c5f595839946c0a71e594e5e889ffc63d85f 100644 --- a/src/views/KuboView.vue +++ b/src/views/KuboView.vue @@ -1,7 +1,7 @@ <script setup lang="ts"> import { ref, type Ref } from 'vue' import { kubo, explorerUrl } from '@/kubo' -import { type StatResult } from 'ipfs-core-types/repo' +// import { type StatResult } from 'ipfs-core-types/repo' import prettyBytes from 'pretty-bytes' const stats: Ref<StatResult> = ref(null)