diff --git a/README.md b/README.md index df9144d3c1cd5f02114daea47ddf46c349f8aa28..16dd0333968ae9c78404419858212bc7d8b87d18 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,4 @@ npx tsx src/scripts/start-indexer.ts ## TODO When using Kubo node, libp2p is not needed at all. +Insert pubkey instead of ss58 address if we want data to be compatible across networks diff --git a/src/interface.ts b/src/interface.ts index 16b5c86d5c7501bc49796b6744e07485ad248be3..5703f166bd8ba49c5726b86bceea297de3755f01 100644 --- a/src/interface.ts +++ b/src/interface.ts @@ -1,5 +1,7 @@ import type { CID } from 'multiformats' import { kubo } from './kubo' +import { emptyInode, type IndexHist, type IndexInode, type IndexLeaf } from './types' +import { IPNS_HIST, BASE, KEYSIZE } from './consts' // interface to easily iterate over the AMT export async function* getAll(cid: CID): AsyncIterable<CID[]> { @@ -19,3 +21,143 @@ export async function* getAll(cid: CID): AsyncIterable<CID[]> { // interface to easily iterate over diff of two AMT export async function* getDiff(cid1: CID, cid2: CID) {} + +// recursive comparison of inodes +// the differences are added asynchronously in "addedLeft" and "addedRight" arrays +// meaning that these arrays will still grow after the function returns +export function compareInodes( + // key of compared inodes, used as termination condition to know if we reached the leaf + k: string, + // reference node for comparison + left: IndexInode, + // node to compare with reference + right: IndexInode, + // list of nodes in left not present in right, (key,value) pairs + addedLeft: Array<[string, CID]>, + // list of nodes in right not present in left, (key,value) pairs + addedRight: Array<[string, CID]> +) { + // termination condition, since we know the size of the key + if (k.length == KEYSIZE) { + console.log('comparing leaf ' + k) + compareLeafs(k, left as unknown as IndexLeaf, right as unknown as IndexLeaf, addedLeft, addedRight) + return + } + // console.log('comparing node ' + k) + + // iterate over nodes children + for (let i = 0; i < BASE; i++) { + // get left and right entries + const li = left.children[i] + const ri = right.children[i] + if (li == null && ri == null) { + // do not compare if they are both null + continue + } + if (li == null) { + // right is not null and was added + const nk = k + ri![0] + console.log('added right ' + nk) + addedRight.push([k, ri![1]]) + continue + } + if (ri == null) { + // left is not null and was added + console.log('added left') + const nk = k + li![0] + addedLeft.push([nk, li![1]]) + continue + } + + // both buckets have items, unstructure them to get key and value + const [lik, lic] = li + const [rik, ric] = ri + if (lic.toString() == ric.toString()) { + // do not compare if the cid is the same + console.log('same ' + k + lik) + continue + } + + if (lik == rik) { + // keys are the same and only content changed, dig deeper in both + const nk = k + lik + Promise.all([kubo.dag.get(lic), kubo.dag.get(ric)]).then((r) => { + const [lin, rin] = r + compareInodes(nk, lin.value, rin.value, addedLeft, addedRight) + }) + continue + } + + // there is a key diff, we have to compare + if (lik.length > rik.length && lik.startsWith(rik)) { + const nk = k + rik + console.log('diff ' + nk) + // intermediate inode might have been added to the right + // create virtual node then dig deeper in right + const lvnode = emptyInode() + const b = parseInt(lik[rik.length], BASE) + lvnode.children[b] = [lik.slice(rik.length), lic] + kubo.dag.get(ric).then((r) => { + compareInodes(nk, lvnode, r.value, addedLeft, addedRight) + }) + continue + } + if (lik.length < rik.length && rik.startsWith(lik)) { + const nk = k + lik + console.log('diff ' + nk) + // intermediate inode might have been added to the left + // create virtual node then dig deeper in right + const rvnode = emptyInode() + const b = parseInt(rik[lik.length], BASE) + rvnode.children[b] = [rik, ric] + kubo.dag.get(ric).then((l) => { + compareInodes(nk, l.value, rvnode, addedLeft, addedRight) + }) + continue + } + // keys do not cover the same time period + // content is then completely different + addedLeft.push([k + lik, lic]) + addedRight.push([k + rik, ric]) + continue + } +} + +// compare leaves by "eating" cids one by one +function compareLeafs( + k: string, + left: IndexLeaf, + right: IndexLeaf, + addedLeft: Array<[string, CID]>, + addedRight: Array<[string, CID]> +) { + while (true) { + const l = left.leaf.shift() + const r = right.leaf.shift() + if (l == undefined) { + right.leaf.map((e) => { + addedRight.push([k, e]) + }) + return + } + if (r == undefined) { + left.leaf.map((e) => { + addedLeft.push([k, e]) + }) + return + } + if (l == r) { + continue + } + if (l < r) { + addedLeft.push([k, l]) + right.leaf.unshift(r) + continue + } + if (l > r) { + addedRight.push([k, r]) + left.leaf.unshift(l) + continue + } + } +} diff --git a/src/main.ts b/src/main.ts index f8cbf195e914cbbb3a66e81b67b7c7b79ec5e9ce..5179324ab1ef7572bd4e73bf6e6df1210037d570 100644 --- a/src/main.ts +++ b/src/main.ts @@ -37,7 +37,7 @@ pubsub.addEventListener('message', (message) => { const isValid = isValidSignature(bytesPayload, dag.sig, dag.pubk) if (isValid) { // here we would do the processing - // addToIndex(cid, dag) + addToIndex(cid, dag) } else { feed.value.push('[invalid sig] ' + msg) } diff --git a/src/views/DiffView.vue b/src/views/DiffView.vue index 0b7809e7dd8b80d5eb4fd0d436deba1dfaeaebf3..6322b425b368b97c5260af5ba3168d9561b19eb7 100644 --- a/src/views/DiffView.vue +++ b/src/views/DiffView.vue @@ -1,9 +1,9 @@ <script setup lang="ts"> import { kubo } from '@/kubo' -import { emptyInode, type IndexHist, type IndexInode, type IndexLeaf } from '../types' -import { IPNS_HIST, BASE, KEYSIZE } from '../consts' +import { IPNS_HIST, KEYSIZE } from '../consts' import { CID } from 'multiformats' import { ref, type Ref, onMounted } from 'vue' +import { compareInodes } from '../interface' const leftMsg = ref('') const rightMsg = ref('') @@ -32,142 +32,13 @@ async function compareRoots() { result.value = 'computing diff...' // fetch root nodes const [leftRoot, rightRoot] = await Promise.all([kubo.dag.get(leftRootCid.value), kubo.dag.get(rightRootCid.value)]) - // compare + // compare inodes, the diff will be added asynchronously in the addedLeft and addedRight arrays compareInodes('', leftRoot.value, rightRoot.value, addedLeft.value, addedRight.value) // print result result.value = 'diff' } -function compareInodes( - k: string, - left: IndexInode, - right: IndexInode, - addedLeft: Array<[string, CID]>, - addedRight: Array<[string, CID]> -) { - if (k.length == KEYSIZE) { - console.log('comparing leaf ' + k) - compareLeafs(k, left as unknown as IndexLeaf, right as unknown as IndexLeaf, addedLeft, addedRight) - return - } - // console.log('comparing node ' + k) - - for (let i = 0; i < BASE; i++) { - // get left and right entries - const li = left.children[i] - const ri = right.children[i] - if (li == null && ri == null) { - // do not compare if they are both null - continue - } - if (li == null) { - // right is not null and was added - const nk = k + ri![0] - console.log('added right ' + nk) - addedRight.push([k, ri![1]]) - continue - } - if (ri == null) { - // left is not null and was added - console.log('added left') - const nk = k + li![0] - addedLeft.push([nk, li![1]]) - continue - } - - // both buckets have items, unstructure them to get key and value - const [lik, lic] = li - const [rik, ric] = ri - if (lic.toString() == ric.toString()) { - // do not compare if the cid is the same - console.log('same ' + k + lik) - continue - } - - if (lik == rik) { - const nk = k + lik - // keys are the same and only content changed, dig deeper in both - Promise.all([kubo.dag.get(lic), kubo.dag.get(ric)]).then((r) => { - const [lin, rin] = r - compareInodes(nk, lin.value, rin.value, addedLeft, addedRight) - }) - continue - } - - // there is a key diff, we have to compare - - if (lik.length > rik.length && lik.startsWith(rik)) { - const nk = k + rik - console.log('diff ' + nk) - // intermediate inode might have been added to the right - // create virtual node then dig deeper in right - const lvnode = emptyInode() - const b = parseInt(lik[rik.length], BASE) - lvnode.children[b] = [lik.slice(rik.length), lic] - kubo.dag.get(ric).then((r) => { - compareInodes(nk, lvnode, r.value, addedLeft, addedRight) - }) - continue - } - if (lik.length < rik.length && rik.startsWith(lik)) { - const nk = k + lik - console.log('diff ' + nk) - // intermediate inode might have been added to the left - // create virtual node then dig deeper in right - const rvnode = emptyInode() - const b = parseInt(rik[lik.length], BASE) - rvnode.children[b] = [rik, ric] - kubo.dag.get(ric).then((l) => { - compareInodes(nk, l.value, rvnode, addedLeft, addedRight) - }) - continue - } - // keys do not cover the same time period - // content is then completely different - addedLeft.push([k + lik, lic]) - addedRight.push([k + rik, ric]) - continue - } -} - -function compareLeafs( - k: string, - left: IndexLeaf, - right: IndexLeaf, - addedLeft: Array<[string, CID]>, - addedRight: Array<[string, CID]> -) { - while (true) { - const l = left.leaf.shift() - const r = right.leaf.shift() - if (l == undefined) { - right.leaf.map((e) => { - addedRight.push([k, e]) - }) - return - } - if (r == undefined) { - left.leaf.map((e) => { - addedLeft.push([k, e]) - }) - return - } - if (l == r) { - continue - } - if (l < r) { - addedLeft.push([k, l]) - right.leaf.unshift(r) - continue - } - if (l > r) { - addedRight.push([k, r]) - left.leaf.unshift(l) - continue - } - } -} - +// init diff comparison with the latest addition to the tree function initWithLastCommit() { async function resolveHist(): Promise<CID> { for await (const name of kubo.name.resolve(IPNS_HIST, { nocache: true })) { @@ -182,6 +53,9 @@ function initWithLastCommit() { if (hist.value.last_history) { kubo.dag.get(hist.value.last_history).then((hist1) => { leftRootCid.value = hist1?.value.current_index + if (leftRootCid.value) { + compareRoots() + } }) } })