diff --git a/Dockerfile.Kubo b/Dockerfile.Kubo index 91317d7606f14c4f36a2192ea6d80359bbae35b3..dab30d6e5a088e11f686cb77559c546246d91928 100644 --- a/Dockerfile.Kubo +++ b/Dockerfile.Kubo @@ -1,4 +1,4 @@ -FROM ipfs/kubo:v0.28.0 +FROM ipfs/kubo:v0.30.0 COPY ./scripts/configure.sh /container-init.d/001-configure.sh CMD ["daemon", "--enable-pubsub-experiment"] # docker buildx build -f Dockerfile.Kubo . -t h30x/datapod-kubo diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 6f01fa0fc856a25edb6ae8c74a2ce294091957d0..7bb1ea73f3bf89690b973d75c01babb777dbe335 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -60,7 +60,7 @@ services: # ------ # optional kubo pubsub to see what the node receives on pubsub pubsub: - image: ipfs/kubo:v0.28.0 + image: ipfs/kubo:v0.30.0 restart: always depends_on: kubo: diff --git a/scripts/configure.sh b/scripts/configure.sh index 31596f32d139aa6df3d7a33369dc56688cf5ad0e..3f6674f4e8c8de511b710eab1bf64614e36c9a26 100644 --- a/scripts/configure.sh +++ b/scripts/configure.sh @@ -58,9 +58,10 @@ ipfs config Peering.Peers --json '[ ipfs config Routing.Type dht # --- rpc --- -# allow easy access through ssh tunnel on port 5002 -# ipfs --api=/ip4/127.0.0.1/tcp/5002 -ipfs config API.HTTPHeaders.Access-Control-Allow-Origin --json '["http://127.0.0.1:5002"]' +# allow easy access through ssh tunnel on port 500x +# ssh -NL 5002:localhost:500x datapod +# ipfs --api=/ip4/127.0.0.1/tcp/500x +ipfs config API.HTTPHeaders.Access-Control-Allow-Origin --json '["http://127.0.0.1:5001","http://127.0.0.1:5002","http://127.0.0.1:5003","http://127.0.0.1:5004","http://127.0.0.1:5005"]' ipfs config API.HTTPHeaders.Access-Control-Allow-Methods --json '["PUT", "POST"]' # --- gateway --- @@ -79,3 +80,9 @@ ipfs config Gateway.ExposeRoutingAPI --json true # only reprovide pinned data # ipfs config Reprovider.Strategy "pinned" # ipfs config Reprovider.Strategy --json null + +# --- IPNS --- +# use pubsub for IPNS records +# ipfs config --json Ipns.UsePubsub true +# republish records frequently +ipfs config --json Ipns.RepublishPeriod '"1min"' \ No newline at end of file diff --git a/scripts/docker-build.sh b/scripts/docker-build.sh index cc5ca3609cf4de749d752aadae10392850d9dee7..03240da301ecd958c19457b5350c0d23967b88ad 100755 --- a/scripts/docker-build.sh +++ b/scripts/docker-build.sh @@ -15,24 +15,24 @@ docker image tag duniter-datapod h30x/duniter-datapod:latest docker image push h30x/duniter-datapod:$version_tag docker image push h30x/duniter-datapod:latest -# # --- kubo -# docker buildx build -f Dockerfile.Kubo -t datapod-kubo . +# --- kubo +docker buildx build -f Dockerfile.Kubo -t datapod-kubo . -# # Tag with version and 'latest' -# docker image tag datapod-kubo h30x/datapod-kubo:$version_tag -# docker image tag datapod-kubo h30x/datapod-kubo:latest +# Tag with version and 'latest' +docker image tag datapod-kubo h30x/datapod-kubo:$version_tag +docker image tag datapod-kubo h30x/datapod-kubo:latest -# # Push both -# docker image push h30x/datapod-kubo:$version_tag -# docker image push h30x/datapod-kubo:latest +# Push both +docker image push h30x/datapod-kubo:$version_tag +docker image push h30x/datapod-kubo:latest -# --- hasura -docker buildx build -f Dockerfile.Hasura -t datapod-hasura . +# # --- hasura +# docker buildx build -f Dockerfile.Hasura -t datapod-hasura . -# Tag with version and 'latest' -docker image tag datapod-hasura h30x/datapod-hasura:$version_tag -docker image tag datapod-hasura h30x/datapod-hasura:latest +# # Tag with version and 'latest' +# docker image tag datapod-hasura h30x/datapod-hasura:$version_tag +# docker image tag datapod-hasura h30x/datapod-hasura:latest -# Push both -docker image push h30x/datapod-hasura:$version_tag -docker image push h30x/datapod-hasura:latest \ No newline at end of file +# # Push both +# docker image push h30x/datapod-hasura:$version_tag +# docker image push h30x/datapod-hasura:latest \ No newline at end of file diff --git a/src/indexer/start.ts b/src/indexer/start.ts index 6627761497ccdaaf6f776d421e8169f01594291a..5bb2b5bf934967391651bdb119db6d37f0529748 100644 --- a/src/indexer/start.ts +++ b/src/indexer/start.ts @@ -103,9 +103,10 @@ const SECOND = 1000 // 1 second const MINUTE = 60 * SECOND // 1 minute const HOUR = 60 * MINUTE // 1 hour const DAY = 24 * HOUR // 1 day +const _ = DAY // ignore unused const HIST_PUBLISH_PERIOD = 10 * MINUTE // regularly sync form peers to compensate from network outage -const PEERSYNC_PERIOD = 1 * DAY +const PEERSYNC_PERIOD = 10 * MINUTE // init globals // set global rootCID from CLI args diff --git a/src/indexer/utils.ts b/src/indexer/utils.ts index 70c608bac2a9eb27f416482f262e324c6e981ea7..b7de7683e23b7120964d2011b60c0f039ce4c7e2 100644 --- a/src/indexer/utils.ts +++ b/src/indexer/utils.ts @@ -3,6 +3,7 @@ import { CID } from 'multiformats' import { kubo } from '../kubo' import { ddKeys } from './ipns' import { getLatestIndexedCID } from './database' +import { emptyRootInode } from '../types' // get root cid from arg if given // initialize it: @@ -44,7 +45,8 @@ export async function getRootCIDfromArgs(argv: string[]): Promise<CID> { // 1. reading from self-published tree const self_bootstrap = ddKeys.tamt try { - for await (const name of kubo.name.resolve(self_bootstrap, { nocache: true })) { + // timeout because resolving local ipns + for await (const name of kubo.name.resolve(self_bootstrap, { timeout: 1000, nocache: true })) { const cid = CID.parse(name.slice(6)) console.log(`🔨 using ${cid} as startup root node`) console.log(` resolved from self ${self_bootstrap}`) @@ -55,8 +57,15 @@ export async function getRootCIDfromArgs(argv: string[]): Promise<CID> { } // 2. if this failed, reads from db (for example if ipfs node was reinitialized but db persisted) const latestCID = await getLatestIndexedCID() - if (latestCID) return latestCID + if (latestCID) { + console.log(`🔨 using ${latestCID} as startup root node`) + console.log(` resolved from self database`) + return latestCID + } // 3. else, starts from scratch, data will be imported from peers later on - console.log('🔨 starting from scratch:', EMPTY_NODE_CID) + console.log(`🔨 starting from scratch: ${EMPTY_NODE_CID}`) + console.log('🌱 adding empty node to ipfs') + const node = emptyRootInode() + await kubo.dag.put(node, { pin: true }) return EMPTY_NODE_CID } diff --git a/src/interface.ts b/src/interface.ts index cac7bdf433de9ef83f463dcbbdf6f2d6a980a626..556ee3ea2757c0e9abad1d3079e6bb0aa16ee009 100644 --- a/src/interface.ts +++ b/src/interface.ts @@ -39,7 +39,7 @@ export async function* getDiff(cid1: CID, cid2: CID): AsyncIterable<CID[]> { if (leaf1.leaf && leaf2.leaf) { const [added1, added2] = compareLeafs('', leaf1, leaf2, [], []) // only look on what was added in cid2 - if (added1.length != 0) console.log('ignoring removed data ' + added1) + if (added1.length != 0) console.log('ignoring missing index request ' + added1) else yield added2.map(([_k, v]) => v) } else if (inode1.children && inode2.children) { // do the inode comparison @@ -70,7 +70,7 @@ async function* getDiffInodes(inode1: IndexInode, inode2: IndexInode): AsyncIter } if (ri == null) { // left is not null and was added, ignore - console.log(`ignoring removed data at ${ctx}${li[0]}: ${li[1]}`) + console.log(`ignoring missing data at ${ctx}${li[0]}: ${li[1]}`) continue } diff --git a/src/processor.ts b/src/processor.ts index 773eb53f951fb1e33493dab89d3b1a70bdac8ead..929109530480e7bdc18bcbf69c339a8de850db0a 100644 --- a/src/processor.ts +++ b/src/processor.ts @@ -202,7 +202,7 @@ export async function publishHistory(cid: CID): Promise<void> { export async function mergeInodesSyncCID(nodeACID: CID, nodeB: IndexVinode | IndexLeaf): Promise<[number, CID]> { // fail with small timeout since this data is supposed to be pinned locally // console.log('fetching ' + nodeACID) - const nodeA = (await kubo.dag.get(nodeACID, { timeout: 1000 })).value // FIXME decrease this timeout without bug + const nodeA = (await kubo.dag.get(nodeACID)).value // FIXME decrease this timeout without bug const newCID = await mergeInodesSync(nodeA, nodeB) // unpin old node CID if different // we do not mind if it was not pinned diff --git a/src/scripts/diff.ts b/src/scripts/diff.ts index 0faeae040afa50639dfe51fb94a8bbc35d3feb5f..97c4aeb59a902ef5dd4c7ec3c338c825cf634bc6 100644 --- a/src/scripts/diff.ts +++ b/src/scripts/diff.ts @@ -8,8 +8,11 @@ console.log('start') // const fromCID = EMPTY_NODE_CID // const fromCID = CID.parse('bafyreic6qy2k5w6324uayfzoybmzypdv57zk3zxezaiws4h553jjogw6o4') // const toCID = CID.parse('bafyreihls2kmwx2ufuwx4kbl67f3ipl5wbc6j6snfegy3sttymrhxsgvpa') -const fromCID = CID.parse('bafyreiaixvejxrcszzexohdo5obtduw5mctvti2jsgob4pnq7ovd5ngrxi') -const toCID = CID.parse('bafyreiel7fh42ehswlh7wg4mz5zzrugwbpuevzojxuzw3dwgyoiyhvjhma') +// const fromCID = CID.parse('bafyreiaixvejxrcszzexohdo5obtduw5mctvti2jsgob4pnq7ovd5ngrxi') +// const toCID = CID.parse('bafyreiel7fh42ehswlh7wg4mz5zzrugwbpuevzojxuzw3dwgyoiyhvjhma') +const fromCID = CID.parse('bafyreifqojtso7fkz2qg3d5p432jbcjgzah2bmvcvowfhbvn4dowghlq2a') +// const toCID = CID.parse('bafyreiel7fh42ehswlh7wg4mz5zzrugwbpuevzojxuzw3dwgyoiyhvjhma') +const toCID = CID.parse('bafyreihwmnq3wtfgbe7mlbwkztasqfcr5auopxwgmw6bnmobhfccgolayu') const iterator = getDiff(fromCID, toCID) diff --git a/src/scripts/emptyRootCID.ts b/src/scripts/emptyRootCID.ts index dfb3927456a298884a93d14e459dfe12b3e760c1..93a7a3b76eb13bef3bb20d9a1203436db5d795da 100644 --- a/src/scripts/emptyRootCID.ts +++ b/src/scripts/emptyRootCID.ts @@ -1,9 +1,9 @@ import { EMPTY_NODE_CID } from '../consts' import { kubo } from '../kubo' import { concretizeCid } from '../processor' -import { emptyInode, emptyVinode } from '../types' +import { emptyRootInode, emptyVinode } from '../types' -const node = emptyInode('') +const node = emptyRootInode() kubo.dag.put(node).then((cid) => console.log('node cid', cid)) const vnode = emptyVinode('') diff --git a/src/scripts/peering.ts b/src/scripts/peering.ts new file mode 100644 index 0000000000000000000000000000000000000000..9513dca6e75e37d5028be37e1c1a748b66919d33 --- /dev/null +++ b/src/scripts/peering.ts @@ -0,0 +1,23 @@ +import { RoutingEventTypes, type RoutingFinalPeerEvent } from 'kubo-rpc-client' +import { kubo } from '../kubo' + +// tests resolution of peers to prototype peer discovery + +async function doit() { + const peers = await kubo.swarm.peers() + for (const p of peers) { + for await (const evt of kubo.routing.findPeer(p.peer)) { + if (evt.type == RoutingEventTypes.FINAL_PEER) { + const e = evt as RoutingFinalPeerEvent + for (const a of e.peer.multiaddrs) { + const astr = a.toString() + if (astr.startsWith('/dns')) { + console.log(astr) + } + } + } + } + } +} + +doit() diff --git a/src/types.ts b/src/types.ts index 0359932f9e6c30c1168ed894b3b947b66e688f71..a2acd3b6ec19a2ee82de8163c4e00afb2b5f54b7 100644 --- a/src/types.ts +++ b/src/types.ts @@ -52,6 +52,10 @@ export function emptyInode(ctx: string): IndexInode { count: 0 } } +// root node has no ctx +export function emptyRootInode(): IndexInode { + return emptyInode('') +} // virtual internal node // same as IndexInode but mutable and only in memory @@ -65,7 +69,7 @@ export interface IndexVinode { export function emptyVinode(ctx: string): IndexVinode { return { children: new Array(BASE).fill(null), - ctx, + ctx } }