diff --git a/src/cesium-plus.ts b/src/cesium-plus.ts
index 5ca1a1cd9b48509b3a1fb603a44e460897aa9b38..6f250354b34f181859f74c9eb2eb872b0cd4ae63 100644
--- a/src/cesium-plus.ts
+++ b/src/cesium-plus.ts
@@ -1,6 +1,8 @@
import { CID } from 'multiformats'
import { kubo } from './kubo'
import { Buffer } from 'buffer'
+import { insertKnownIndexRequest } from './processor'
+import type { IndexRequest, Pointer } from './types'
// for reference see
// https://doc.e-is.pro/cesium-plus-pod/REST_API.html
@@ -55,3 +57,52 @@ export async function processCesiumPlusProfile(obj: CplusProfile): Promise<CID>
return kubo.dag.put(profile) as Promise<CID>
}
}
+
+// import these cid to target AMT
+export async function importCplusToAMT(cplusCID: CID, amtCid: Pointer<CID>) {
+ const cplusroot = await kubo.dag.get(cplusCID)
+ for (let chunkcid of cplusroot.value) {
+ // process each chunk sequentially to avoid memory overflow
+ const chunk = await kubo.dag.get(chunkcid)
+ chunk.value.map((pcid: CID) =>
+ kubo.dag.get(pcid).then((p) => {
+ const profile: CplusProfile = p.value
+ const indexRequest: IndexRequest = {
+ pubkey: profile.issuer,
+ cid: pcid,
+ timestamp: profile.time,
+ signature: '' // signature is inside document for C+ data
+ }
+ kubo.dag.put(indexRequest).then(async (indexRequestCid) => {
+ // update CID synchronously for next insert
+ amtCid.value = await insertKnownIndexRequest(amtCid.value, indexRequestCid as CID, indexRequest)
+ })
+ })
+ )
+ }
+}
+
+// alternative slow synchronous function to avoid overflowing kubo with connections
+// exectued in chromium, it can import about 18 profiles per second
+export async function importCplusToAMTSync(cplusCID: CID, amtCid: CID): Promise<CID> {
+ const cplusroot = await kubo.dag.get(cplusCID)
+ let importedCount = 0
+ for (let chunkcid of cplusroot.value) {
+ const chunk = await kubo.dag.get(chunkcid)
+ for (let pcid of chunk.value) {
+ const p = await kubo.dag.get(pcid)
+ const profile: CplusProfile = p.value
+ const indexRequest: IndexRequest = {
+ pubkey: profile.issuer,
+ cid: pcid,
+ timestamp: profile.time,
+ signature: '' // signature is inside document for C+ data
+ }
+ const indexRequestCid = await kubo.dag.put(indexRequest)
+ amtCid = await insertKnownIndexRequest(amtCid, indexRequestCid as CID, indexRequest)
+ importedCount += 1
+ console.log(importedCount)
+ }
+ }
+ return amtCid
+}
diff --git a/src/processor.ts b/src/processor.ts
index 9a51b3227147e72dc049e0f04929d49043dd8cd2..5eee07a798e422e480922bfc1b37d2a3574fdde2 100644
--- a/src/processor.ts
+++ b/src/processor.ts
@@ -30,11 +30,11 @@ export async function addToIndexQueue(cid: CID, indexRequest: IndexRequest) {
export async function insertIndexRequest(rootCid: CID, indexRequestCid: CID): Promise<CID> {
const [rootDag, indexDag] = await Promise.all([kubo.dag.get(rootCid), kubo.dag.get(indexRequestCid)])
const key = indexDag.value.timestamp.toString(BASE).padStart(KEYSIZE, '0')
- return processInode(rootDag.value, key, indexDag.value)
+ return processInode(rootDag.value, key, indexRequestCid)
}
// same as above but with known request
-async function insertKnownIndexRequest(rootCid: CID, indexRequestCid: CID, indexRequest: IndexRequest): Promise<CID> {
+export async function insertKnownIndexRequest(rootCid: CID, indexRequestCid: CID, indexRequest: IndexRequest): Promise<CID> {
const key = indexRequest.timestamp.toString(BASE).padStart(KEYSIZE, '0')
const rootDag = await kubo.dag.get(rootCid)
return processInode(rootDag.value, key, indexRequestCid)
diff --git a/src/scripts/cesium-plus-import.ts b/src/scripts/cesium-plus-import.ts
index 295abd302cb3f53064392ecab2f9d4007ead2ffb..f3b3067be53405497fee3e286f3e8b12fdc79885 100644
--- a/src/scripts/cesium-plus-import.ts
+++ b/src/scripts/cesium-plus-import.ts
@@ -1,5 +1,5 @@
-import type { CID } from 'multiformats'
-import { processCesiumPlusImport, processCesiumPlusProfile } from '../cesium-plus'
+import { CID } from 'multiformats'
+import { processCesiumPlusImport, processCesiumPlusProfile, importCplusToAMTSync } from '../cesium-plus'
import * as fs from 'fs/promises'
// profile files
@@ -23,4 +23,14 @@ async function doImport() {
processCesiumPlusImport(cids, GROUPBY).then((cid) => console.log(cid))
}
-doImport()
+// doImport()
+
+// in node, this is a bit faster than in chromium, and we get about 25 profiles per second instead of 18
+// but this shows we should optimise AMT inserting for higher throughput
+function doImportToAMT() {
+ const cplus = CID.parse('bafyreie74jtf23zzz2tdgsz7axfrm4pidje43ypqn25v4gkdtfjbcj62km') // cesium plus import
+ const amt = CID.parse('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid
+ importCplusToAMTSync(cplus, amt).then(console.log)
+}
+
+doImportToAMT()
\ No newline at end of file
diff --git a/src/types.ts b/src/types.ts
index 09c103a8aa3e04ac84fa49e883e42e59a726df74..0cfd81b01b56faf6ce8c0f9401538de697668065 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -1,6 +1,10 @@
import { CID } from 'multiformats'
import { BASE } from './consts'
+export interface Pointer<T> {
+ value: T
+}
+
export interface IndexRequest {
pubkey: string
cid: CID
diff --git a/src/views/CplusView.vue b/src/views/CplusView.vue
index e4cd5ba7060e2a0bf9aaefe2a3a31d022ea1a462..f49a4daf55d3a0bf4366a827f20e372a48f5b8ab 100644
--- a/src/views/CplusView.vue
+++ b/src/views/CplusView.vue
@@ -1,16 +1,33 @@
<script setup lang="ts">
-import { ref, type Ref, computed } from 'vue'
-import { processCesiumPlusProfile, type CplusProfile } from '../cesium-plus'
+import { ref, type Ref, onMounted } from 'vue'
+import { processCesiumPlusProfile, type CplusProfile, importCplusToAMTSync } from '../cesium-plus'
+import { CID } from 'multiformats'
const file: Ref<File | null> = ref(null)
const sample = ref('')
const cidlist = ref('')
const MAX = 20
-
+const leftMsg = ref('bafyreie74jtf23zzz2tdgsz7axfrm4pidje43ypqn25v4gkdtfjbcj62km') // cesium plus import
+const rightMsg = ref('bafyreicvlp2p65agkxpzcboedba7zit55us4zvtyyq2wesvsdedy6irwfy') // empty root cid
+const leftRootCid: Ref<CID | null> = ref(null)
+const rightRootCid: Ref<CID | null> = ref(null)
+function setLeftRoot() {
+ leftRootCid.value = CID.parse(leftMsg.value)
+}
+function setRightRoot() {
+ rightRootCid.value = CID.parse(rightMsg.value)
+}
function setfile(input: any) {
file.value = input.target.files[0]
}
+onMounted(() => {
+ try {
+ setLeftRoot()
+ setRightRoot()
+ } catch {}
+})
+
function importCplus() {
cidlist.value = ''
if (file.value) {
@@ -27,6 +44,12 @@ function importCplus() {
reader.readAsText(file.value)
}
}
+
+function doImportCplusToAMT() {
+ if (leftRootCid.value != null && rightRootCid.value != null) {
+ importCplusToAMTSync(leftRootCid.value, rightRootCid.value).then((c) => (rightRootCid.value = c))
+ }
+}
</script>
<template>
@@ -47,6 +70,19 @@ function importCplus() {
<h2>Import from root cid</h2>
<p>Choose a root CID from Cesium import and the root CID of the target AMT</p>
+ <p>
+ <input v-model="leftMsg" @keyup.enter="setLeftRoot" />
+ <button v-on:click="setLeftRoot">set cesium root</button>
+ <input v-model="rightMsg" @keyup.enter="setRightRoot" />
+ <button v-on:click="setRightRoot">set target AMT</button>
+ </p>
+ <p class="mono">{{ leftRootCid }} {{ rightRootCid }}</p>
+ <p>
+ <button v-on:click="doImportCplusToAMT">import !</button>
+ </p>
+ <p>
+ {{ rightRootCid?.toString() }}
+ </p>
</template>
<style scoped></style>
diff --git a/src/views/DiffView.vue b/src/views/DiffView.vue
index 0ca51a1f704e29dc6f1f61b9b1cdbfa961dd000f..0b7809e7dd8b80d5eb4fd0d436deba1dfaeaebf3 100644
--- a/src/views/DiffView.vue
+++ b/src/views/DiffView.vue
@@ -3,7 +3,7 @@ import { kubo } from '@/kubo'
import { emptyInode, type IndexHist, type IndexInode, type IndexLeaf } from '../types'
import { IPNS_HIST, BASE, KEYSIZE } from '../consts'
import { CID } from 'multiformats'
-import { ref, type Ref, computed } from 'vue'
+import { ref, type Ref, onMounted } from 'vue'
const leftMsg = ref('')
const rightMsg = ref('')
@@ -179,14 +179,16 @@ function initWithLastCommit() {
resolveHist().then((histCid) => {
kubo.dag.get(histCid).then((hist) => {
rightRootCid.value = hist.value.current_index
- kubo.dag.get(hist.value.last_history).then((hist1) => {
- leftRootCid.value = hist1?.value.current_index
- })
+ if (hist.value.last_history) {
+ kubo.dag.get(hist.value.last_history).then((hist1) => {
+ leftRootCid.value = hist1?.value.current_index
+ })
+ }
})
})
}
-initWithLastCommit()
+onMounted(initWithLastCommit)
</script>
<template>
diff --git a/src/views/IndexView.vue b/src/views/IndexView.vue
index cdb9a468b76fe5b859fac930ba0df28188a2ded1..fdfdd5229730af86efc08dc944fe8b7d92bc2c38 100644
--- a/src/views/IndexView.vue
+++ b/src/views/IndexView.vue
@@ -3,7 +3,7 @@ import { kubo } from '@/kubo'
import { emptyInode, type IndexHist } from '../types'
import { IPNS, IPNS_HIST } from '../consts'
import { CID } from 'multiformats'
-import { ref, type Ref, computed } from 'vue'
+import { ref, type Ref, computed, onMounted } from 'vue'
import IndexNode from '../components/IndexNode.vue'
import IndexHistory from '../components/IndexHistory.vue'
@@ -35,7 +35,7 @@ async function initIndex() {
// resolve again to confirm publish result
resolve()
// update history accordingly
- updateHist(rootcid)
+ updateHist(rootcid as CID)
}
// inits indexing and publishes ipns entry for it
@@ -54,7 +54,6 @@ async function initIndexHist() {
async function resolve() {
for await (const name of kubo.name.resolve(IPNS, { nocache: true })) {
- console.log(name)
ipnsTarget.value = name
targetCid.value = CID.parse(name.slice(6))
}
@@ -91,8 +90,10 @@ function setTargetCid() {
}
}
-resolve()
-resolveHist()
+onMounted(() => {
+ resolve()
+ resolveHist()
+})
</script>
<template>
diff --git a/src/views/KuboView.vue b/src/views/KuboView.vue
index b2969abb9c04b892892867ee0bcfd09ea5b3d2bd..dd12c5f595839946c0a71e594e5e889ffc63d85f 100644
--- a/src/views/KuboView.vue
+++ b/src/views/KuboView.vue
@@ -1,7 +1,7 @@
<script setup lang="ts">
import { ref, type Ref } from 'vue'
import { kubo, explorerUrl } from '@/kubo'
-import { type StatResult } from 'ipfs-core-types/repo'
+// import { type StatResult } from 'ipfs-core-types/repo'
import prettyBytes from 'pretty-bytes'
const stats: Ref<StatResult> = ref(null)