diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts index 3814d5aca148f6388bc259cf74423a532306848b..2175e3eb1a1d203cf3791a849d479770682523da 100644 --- a/app/modules/crawler/index.ts +++ b/app/modules/crawler/index.ts @@ -26,7 +26,7 @@ import {FileDAL} from "../../lib/dal/fileDAL" import {RemoteSynchronizer} from "./lib/sync/RemoteSynchronizer" import {AbstractSynchronizer} from "./lib/sync/AbstractSynchronizer" import {LocalPathSynchronizer} from "./lib/sync/LocalPathSynchronizer" -import {CommonConstants} from "../../lib/common-libs/constants"; +import {CommonConstants} from "../../lib/common-libs/constants" export const CrawlerDependency = { duniter: { @@ -49,8 +49,8 @@ export const CrawlerDependency = { return crawler.sandboxPull(server) }, - synchronize: (server:Server, onHost:string, onPort:number, upTo:number, chunkLength:number) => { - const strategy = new RemoteSynchronizer(onHost, onPort, server, chunkLength) + synchronize: (server:Server, onHost:string, onPort:number, upTo:number, chunkLength:number, allowLocalSync = false) => { + const strategy = new RemoteSynchronizer(onHost, onPort, server, chunkLength, undefined, undefined, allowLocalSync) const remote = new Synchroniser(server, strategy) const syncPromise = remote.sync(upTo, chunkLength) return { @@ -77,6 +77,7 @@ export const CrawlerDependency = { { value: '--cautious', desc: 'Check blocks validity during sync (overrides --nocautious option).'}, { value: '--nopeers', desc: 'Do not retrieve peers during sync.'}, { value: '--nop2p', desc: 'Disables P2P downloading of blocs during sync.'}, + {Â value: '--localsync', desc: 'Allow to synchronize on nodes with local network IP address for `sync` command' }, { value: '--nosources', desc: 'Do not parse sources (UD, TX) during sync (debug purposes).'}, { value: '--nosbx', desc: 'Do not retrieve sandboxes during sync.'}, { value: '--onlypeers', desc: 'Will only try to sync peers.'}, @@ -127,7 +128,7 @@ export const CrawlerDependency = { const sp = source.split(':') const onHost = sp[0] const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443 - strategy = new RemoteSynchronizer(onHost, onPort, server, CommonConstants.SYNC_BLOCKS_CHUNK, noShufflePeers === true, otherDAL) + strategy = new RemoteSynchronizer(onHost, onPort, server, CommonConstants.SYNC_BLOCKS_CHUNK, noShufflePeers === true, otherDAL, program.localsync !== undefined) } else { strategy = new LocalPathSynchronizer(source, server, CommonConstants.SYNC_BLOCKS_CHUNK) } diff --git a/app/modules/crawler/lib/sync/P2PSyncDownloader.ts b/app/modules/crawler/lib/sync/P2PSyncDownloader.ts index af8987341be8179b075a656a6b666d57edc14105..c8bc97fae388c8a6264053dafefbbdbcf8fdc672 100644 --- a/app/modules/crawler/lib/sync/P2PSyncDownloader.ts +++ b/app/modules/crawler/lib/sync/P2PSyncDownloader.ts @@ -7,7 +7,7 @@ import {ISyncDownloader} from "./ISyncDownloader" import {cliprogram} from "../../../../lib/common-libs/programOptions" import {Keypair} from "../../../../lib/dto/ConfDTO" import {IRemoteContacter} from "./IRemoteContacter" -import {ManualPromise, newManualPromise} from "../../../../lib/common-libs/manual-promise" +import {ManualPromise} from "../../../../lib/common-libs/manual-promise" import {GlobalFifoPromise} from "../../../../service/GlobalFifoPromise" import {getNanosecondsTime} from "../../../../ProcessCpuProfiler" import {CommonConstants} from "../../../../lib/common-libs/constants" @@ -40,6 +40,7 @@ export class P2PSyncDownloader extends ASyncDownloader implements ISyncDownloade private watcher:Watcher, private logger:any, public chunkSize: number, + public allowLocalSync: boolean, ) { super(chunkSize) this.TOO_LONG_TIME_DOWNLOAD = "No answer after " + this.MAX_DELAY_PER_DOWNLOAD + "ms, will retry download later."; @@ -48,7 +49,7 @@ export class P2PSyncDownloader extends ASyncDownloader implements ISyncDownloade this.processing = Array.from({ length: this.numberOfChunksToDownload }).map(() => false); this.handler = Array.from({ length: this.numberOfChunksToDownload }).map(() => null); - this.p2pCandidates = peers.map(p => new P2pCandidate(PeerDTO.fromJSONObject(p), this.keypair, this.logger)) + this.p2pCandidates = peers.map(p => new P2pCandidate(PeerDTO.fromJSONObject(p), this.keypair, this.logger, allowLocalSync)) } get maxSlots(): number { diff --git a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts index 666a4c455feb3910589c2636236ded5a24675577..0d33f7977ddced0baca9326c254a1184e0805ccf 100644 --- a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts +++ b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts @@ -28,17 +28,16 @@ import {FsSyncDownloader} from "./FsSyncDownloader" import {AbstractSynchronizer} from "./AbstractSynchronizer" import {pullSandboxToLocalServer} from "../sandbox" import * as path from 'path' -import {IRemoteContacter} from "./IRemoteContacter"; -import {BMARemoteContacter} from "./BMARemoteContacter"; -import {WS2PConnection, WS2PPubkeyRemoteAuth, WS2PPubkeySyncLocalAuth} from "../../../ws2p/lib/WS2PConnection"; -import {WS2PRequester} from "../../../ws2p/lib/WS2PRequester"; -import {WS2PMessageHandler} from "../../../ws2p/lib/impl/WS2PMessageHandler"; -import {WS2PResponse} from "../../../ws2p/lib/impl/WS2PResponse"; -import {DataErrors} from "../../../../lib/common-libs/errors"; -import {KeyGen} from "../../../../lib/common-libs/crypto/keyring"; -import {WS2PRemoteContacter} from "./WS2PRemoteContacter"; -import {Keypair} from "../../../../lib/dto/ConfDTO"; -import {cat} from "shelljs"; +import {IRemoteContacter} from "./IRemoteContacter" +import {BMARemoteContacter} from "./BMARemoteContacter" +import {WS2PConnection, WS2PPubkeyRemoteAuth, WS2PPubkeySyncLocalAuth} from "../../../ws2p/lib/WS2PConnection" +import {WS2PRequester} from "../../../ws2p/lib/WS2PRequester" +import {WS2PMessageHandler} from "../../../ws2p/lib/impl/WS2PMessageHandler" +import {WS2PResponse} from "../../../ws2p/lib/impl/WS2PResponse" +import {DataErrors} from "../../../../lib/common-libs/errors" +import {KeyGen} from "../../../../lib/common-libs/crypto/keyring" +import {WS2PRemoteContacter} from "./WS2PRemoteContacter" +import {Keypair} from "../../../../lib/dto/ConfDTO" const logger = NewLogger() @@ -62,6 +61,7 @@ export class RemoteSynchronizer extends AbstractSynchronizer { chunkSize: number, private noShufflePeers = false, private otherDAL?:FileDAL, + private allowLocalSync = false, ) { super(chunkSize) } @@ -227,7 +227,7 @@ export class RemoteSynchronizer extends AbstractSynchronizer { p2pDownloader(): ISyncDownloader { if (!this.theP2pDownloader) { - this.theP2pDownloader = new P2PSyncDownloader(this.currency, this.server.conf.pair, this.localNumber, this.to, this.shuffledPeers, this.watcher, logger, this.chunkSize) + this.theP2pDownloader = new P2PSyncDownloader(this.currency, this.server.conf.pair, this.localNumber, this.to, this.shuffledPeers, this.watcher, logger, this.chunkSize, this.allowLocalSync) } return this.theP2pDownloader } diff --git a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts index e5dc9f025a63ff9d9dd754db70626cc0ba68fea2..4ece5e23313db13af861c935f6d609ece58c5915 100644 --- a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts +++ b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts @@ -19,7 +19,8 @@ export class P2pCandidate { constructor( private p: PeerDTO, private keypair: Keypair, - private logger: any + private logger: any, + private allowLocalSync: boolean, ) { this.apiPromise = this.initAPI() this.dlPromise = querablep(Promise.resolve(null)) @@ -114,7 +115,7 @@ export class P2pCandidate { try { const apis = this.getRemoteAPIs() const syncApi = await RemoteSynchronizer.getSyncAPI(apis, this.keypair) - if ((syncApi && syncApi.api.hostName || '').match(/^(localhost|192|127)/)) { + if (!this.allowLocalSync && ((syncApi && syncApi.api.hostName || '').match(/^(localhost|192|127)/))) { return null } this.api = syncApi.api diff --git a/test/integration/misc/cli.ts b/test/integration/misc/cli.ts index 4a02c6a5e13dcade7564d3b65a67b89680d584a3..7b58df1b3103f6872859ee615358af48c9c1c905 100644 --- a/test/integration/misc/cli.ts +++ b/test/integration/misc/cli.ts @@ -126,7 +126,7 @@ describe("CLI", function() { it('sync 7 blocks (fast)', async () => { // await execute(['reset', 'data']); - await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '--nocautious', '--nointeractive', '--noshuffle', '7']); + await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '--nocautious', '--nointeractive', '--noshuffle', '--localsync', '7']); const res = await execute(['export-bc', '--nostdout']); res[res.length - 1].should.have.property('number').equal(7); res.should.have.length(7 + 1); // blocks #0..#7 diff --git a/test/integration/ws2p/ws2p_sync.ts b/test/integration/ws2p/ws2p_sync.ts index 2e1e65018c195055dc56c48e8dfa794c8cf7bb2c..faa75739b062a4be76dcbbfc748902d38da5e8e2 100644 --- a/test/integration/ws2p/ws2p_sync.ts +++ b/test/integration/ws2p/ws2p_sync.ts @@ -13,8 +13,8 @@ import {WS2PConstants} from "../../../app/modules/ws2p/lib/constants" import {assertEqual, assertNotNull, createCurrencyWith2Blocks, writeBasicTestWith2Users} from "../tools/test-framework" -import {NewTestingServer, TestWS2PAPI} from "../tools/toolbox"; -import {CrawlerDependency} from "../../../app/modules/crawler/index"; +import {NewTestingServer, TestWS2PAPI} from "../tools/toolbox" +import {CrawlerDependency} from "../../../app/modules/crawler/index" describe('WS2P sync', () => writeBasicTestWith2Users((test) => { @@ -54,7 +54,7 @@ describe('WS2P sync', () => writeBasicTestWith2Users((test) => { const s2 = NewTestingServer({ pair: cat.keypair }) await s2.initWithDAL() // We sync on s1 - await CrawlerDependency.duniter.methods.synchronize(s2._server, ws2p.host, ws2p.port, 2, 2).syncPromise + await CrawlerDependency.duniter.methods.synchronize(s2._server, ws2p.host, ws2p.port, 2, 2, true).syncPromise assertNotNull(await s2.dal.getCurrentBlockOrNull()) }) }))