diff --git a/app/lib/common-libs/errors.ts b/app/lib/common-libs/errors.ts
index d44e0f6eadc87a6f1b7594e9e753251c3735a6fa..358152eda0d8b52502a3ebc0fd220d488fa7cc19 100755
--- a/app/lib/common-libs/errors.ts
+++ b/app/lib/common-libs/errors.ts
@@ -1,5 +1,9 @@
export enum DataErrors {
+ WRONG_CURRENCY_DETECTED,
+ NO_PEERING_AVAILABLE_FOR_SYNC,
+ REMOTE_HAS_NO_CURRENT_BLOCK,
+ CANNOT_CONNECT_TO_REMOTE_FOR_SYNC,
WS2P_SYNC_PERIMETER_IS_LIMITED,
PEER_REJECTED,
TOO_OLD_PEER,
diff --git a/app/lib/other_constants.ts b/app/lib/other_constants.ts
index 2b0139b508162b43c0d3948f0738687386b8d16b..3f62a541231e5ccea2efc85a39e1084bec9d360d 100644
--- a/app/lib/other_constants.ts
+++ b/app/lib/other_constants.ts
@@ -13,7 +13,7 @@
export const OtherConstants = {
- MUTE_LOGS_DURING_UNIT_TESTS: true,
+ MUTE_LOGS_DURING_UNIT_TESTS: false,
SQL_TRACES: false,
BC_EVENT: {
diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts
index c2b5ffe7e5456d8ab2329d833c69c03666abea5d..b6ccfe1b8dfe406118f3b4ea712fb81ea75d1717 100644
--- a/app/modules/crawler/index.ts
+++ b/app/modules/crawler/index.ts
@@ -48,8 +48,8 @@ export const CrawlerDependency = {
return crawler.sandboxPull(server)
},
- synchronize: (server:Server, onHost:string, onPort:number, upTo:number, chunkLength:number) => {
- const strategy = new RemoteSynchronizer(onHost, onPort, server)
+ synchronize: (currency: string, server:Server, onHost:string, onPort:number, upTo:number, chunkLength:number) => {
+ const strategy = new RemoteSynchronizer(currency, onHost, onPort, server)
const remote = new Synchroniser(server, strategy)
const syncPromise = (async () => {
await server.dal.disableChangesAPI()
@@ -69,8 +69,8 @@ export const CrawlerDependency = {
* @param {number} onPort
* @returns {Promise<any>}
*/
- testForSync: (server:Server, onHost:string, onPort:number) => {
- return RemoteSynchronizer.test(onHost, onPort)
+ testForSync: (currency: string, server:Server, onHost:string, onPort:number) => {
+ return RemoteSynchronizer.test(currency, onHost, onPort, server.conf.pair)
}
},
@@ -88,12 +88,13 @@ export const CrawlerDependency = {
],
cli: [{
- name: 'sync [source] [to]',
+ name: 'sync [source] [to] [currency]',
desc: 'Synchronize blockchain from a remote Duniter node',
preventIfRunning: true,
onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any): Promise<any> => {
- const source = params[0]
- const to = params[1]
+ const source = params[0]
+ const to = params[1]
+ const currency = params[2]
const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/
const FILE_PATTERN = /^(\/.+)$/
if (!source || !(source.match(HOST_PATTERN) || source.match(FILE_PATTERN))) {
@@ -126,7 +127,10 @@ export const CrawlerDependency = {
const sp = source.split(':')
const onHost = sp[0]
const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443
- strategy = new RemoteSynchronizer(onHost, onPort, server, noShufflePeers === true, otherDAL)
+ if (!currency) {
+ throw 'currency parameter is required for network synchronization'
+ }
+ strategy = new RemoteSynchronizer(currency, onHost, onPort, server, noShufflePeers === true, otherDAL)
} else {
strategy = new LocalPathSynchronizer(source, server)
}
diff --git a/app/modules/crawler/lib/connect.ts b/app/modules/crawler/lib/connect.ts
index 2d89d28a6612e029494173a3ec16390aa2c96011..f87741c5e8c18243d62d17bcd41832c032da1f5e 100644
--- a/app/modules/crawler/lib/connect.ts
+++ b/app/modules/crawler/lib/connect.ts
@@ -13,10 +13,11 @@
import {CrawlerConstants} from "./constants"
import {Contacter} from "./contacter"
+import {PeerDTO} from "../../../lib/dto/PeerDTO";
const DEFAULT_HOST = 'localhost';
-export const connect = (peer:any, timeout:number|null = null) => {
+export const connect = (peer:PeerDTO, timeout:number|null = null) => {
return Promise.resolve(new Contacter(peer.getDns() || peer.getIPv4() || peer.getIPv6() || DEFAULT_HOST, peer.getPort(), {
timeout: timeout || CrawlerConstants.DEFAULT_TIMEOUT
}))
diff --git a/app/modules/crawler/lib/contacter.ts b/app/modules/crawler/lib/contacter.ts
index 7789fc0f6c0da8f65c7dd2d713f0c2430ffc085e..0e493502f6fc2ceabf760d0de03422ec10cc1640 100644
--- a/app/modules/crawler/lib/contacter.ts
+++ b/app/modules/crawler/lib/contacter.ts
@@ -65,6 +65,10 @@ export class Contacter {
getPeers(obj?:any) {
return this.get('/network/peering/peers', dtos.MerkleOfPeers, obj)
}
+
+ getPeersArray() {
+ return this.get('/network/peering/peers', dtos.Peers)
+ }
getSources(pubkey:string) {
return this.get('/tx/sources/', dtos.Sources, pubkey)
diff --git a/app/modules/crawler/lib/sandbox.ts b/app/modules/crawler/lib/sandbox.ts
index f4a53218089179c38b3fd8e93268afa260823986..5a92e8a87c7b0e15ae345dacada964d78cc37476 100644
--- a/app/modules/crawler/lib/sandbox.ts
+++ b/app/modules/crawler/lib/sandbox.ts
@@ -16,6 +16,7 @@ import {Contacter} from "./contacter"
import {Server} from "../../../../server"
import {rawer} from "../../../lib/common-libs/index"
import {parsers} from "../../../lib/common-libs/parsers/index"
+import {IRemoteContacter} from "./sync/IRemoteContacter";
export const pullSandbox = async (currency:string, fromHost:string, fromPort:number, toHost:string, toPort:number, logger:any) => {
const from = new Contacter(fromHost, fromPort);
@@ -43,12 +44,12 @@ export const pullSandbox = async (currency:string, fromHost:string, fromPort:num
}
}
-export const pullSandboxToLocalServer = async (currency:string, fromHost:any, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
+export const pullSandboxToLocalServer = async (currency:string, fromHost:IRemoteContacter, toServer:Server, logger:any, watcher:any = null, nbCertsMin = 1, notify = true) => {
let res
try {
res = await fromHost.getRequirementsPending(nbCertsMin || 1)
} catch (e) {
- watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', [fromHost.host, fromHost.port].join(':'))
+ watcher && watcher.writeStatus('Sandbox pulling: could not fetch requirements on %s', fromHost.getName())
}
if (res) {
diff --git a/app/modules/crawler/lib/sync/BMARemoteContacter.ts b/app/modules/crawler/lib/sync/BMARemoteContacter.ts
new file mode 100644
index 0000000000000000000000000000000000000000..0726cec0b842ae8bf474b3bf85617d70ca3cbde2
--- /dev/null
+++ b/app/modules/crawler/lib/sync/BMARemoteContacter.ts
@@ -0,0 +1,56 @@
+// Source file from duniter: Crypto-currency software to manage libre currency such as Äž1
+// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+import {NewLogger} from "../../../../lib/logger"
+import {IRemoteContacter} from "./IRemoteContacter";
+import {Contacter} from "../contacter";
+import {HttpMerkleOfPeers, HttpRequirements} from "../../../bma/lib/dtos";
+import {JSONDBPeer} from "../../../../lib/db/DBPeer";
+import {FileDAL} from "../../../../lib/dal/fileDAL";
+import {Watcher} from "./Watcher";
+import {cliprogram} from "../../../../lib/common-libs/programOptions";
+import {connect} from "../connect";
+import {RemoteSynchronizer} from "./RemoteSynchronizer";
+import {PeerDTO} from "../../../../lib/dto/PeerDTO";
+import {CrawlerConstants} from "../constants";
+import {dos2unix} from "../../../../lib/common-libs/dos2unix";
+import {PeeringService} from "../../../../service/PeeringService";
+import {BlockDTO} from "../../../../lib/dto/BlockDTO";
+
+const logger = NewLogger()
+
+export class BMARemoteContacter implements IRemoteContacter {
+
+ constructor(protected contacter: Contacter) {
+ }
+
+ getBlock(number: number): Promise<BlockDTO | null> {
+ return this.contacter.getBlock(number)
+ }
+
+ getCurrent(): Promise<BlockDTO | null> {
+ return this.contacter.getCurrent()
+ }
+
+ async getPeers(): Promise<(JSONDBPeer|null)[]> {
+ return (await this.contacter.getPeersArray()).peers
+ }
+
+ getRequirementsPending(minsig: number): Promise<HttpRequirements> {
+ return this.contacter.getRequirementsPending(minsig)
+ }
+
+ getName(): string {
+ return "BMA remote '" + this.contacter.fullyQualifiedHost + "'"
+ }
+}
diff --git a/app/modules/crawler/lib/sync/IRemoteContacter.ts b/app/modules/crawler/lib/sync/IRemoteContacter.ts
new file mode 100644
index 0000000000000000000000000000000000000000..fc3b8c3bc22971cf389e2c9f71043f533265c618
--- /dev/null
+++ b/app/modules/crawler/lib/sync/IRemoteContacter.ts
@@ -0,0 +1,29 @@
+// Source file from duniter: Crypto-currency software to manage libre currency such as Äž1
+// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+import {JSONDBPeer} from "../../../../lib/db/DBPeer";
+import {BlockDTO} from "../../../../lib/dto/BlockDTO";
+import {HttpRequirements} from "../../../bma/lib/dtos";
+
+export interface IRemoteContacter {
+
+ getName(): string
+
+ getPeers(): Promise<(JSONDBPeer|null)[]>
+
+ getCurrent(): Promise<BlockDTO|null>
+
+ getBlock(number: number): Promise<BlockDTO|null>
+
+ getRequirementsPending(number: number): Promise<HttpRequirements>
+}
diff --git a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
index 9767f3de60c15beab8c687d68ced2aadd468cb51..08ef13a4244c171115a964a208744f5d16639375 100644
--- a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
@@ -14,7 +14,6 @@
import {ISyncDownloader} from "./ISyncDownloader"
import {BlockDTO} from "../../../../lib/dto/BlockDTO"
import {PeerDTO} from "../../../../lib/dto/PeerDTO"
-import {Contacter} from "../contacter"
import {connect} from "../connect"
import {NewLogger} from "../../../../lib/logger"
import {CrawlerConstants} from "../constants"
@@ -32,25 +31,37 @@ import {FsSyncDownloader} from "./FsSyncDownloader"
import {AbstractSynchronizer} from "./AbstractSynchronizer"
import {pullSandboxToLocalServer} from "../sandbox"
import * as path from 'path'
+import {IRemoteContacter} from "./IRemoteContacter";
+import {BMARemoteContacter} from "./BMARemoteContacter";
+import {WS2PConnection, WS2PPubkeyLocalAuth, WS2PPubkeyRemoteAuth} from "../../../ws2p/lib/WS2PConnection";
+import {WS2PRequester} from "../../../ws2p/lib/WS2PRequester";
+import {WS2PServerMessageHandler} from "../../../ws2p/lib/interface/WS2PServerMessageHandler";
+import {WS2PMessageHandler} from "../../../ws2p/lib/impl/WS2PMessageHandler";
+import {WS2PResponse} from "../../../ws2p/lib/impl/WS2PResponse";
+import {DataErrors} from "../../../../lib/common-libs/errors";
+import {Key, KeyGen} from "../../../../lib/common-libs/crypto/keyring";
+import {WS2PRemoteContacter} from "./WS2PRemoteContacter";
+import {Keypair} from "../../../../lib/dto/ConfDTO";
+import {cat} from "shelljs";
const logger = NewLogger()
export class RemoteSynchronizer extends AbstractSynchronizer {
- private node:Contacter
+ private node:IRemoteContacter
private peer:PeerDTO
private shuffledPeers: JSONDBPeer[]
private theP2pDownloader: ISyncDownloader
private theFsDownloader: ISyncDownloader
private to: number
private localNumber: number
- private currency: string
private watcher: Watcher
private static contacterOptions = {
timeout: CrawlerConstants.SYNC_LONG_TIMEOUT
}
constructor(
+ private readonly currency: string,
private host: string,
private port: number,
private server:Server,
@@ -89,54 +100,75 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
}
async init(): Promise<void> {
- const peering = await Contacter.fetchPeer(this.host, this.port, RemoteSynchronizer.contacterOptions)
- this.peer = PeerDTO.fromJSONObject(peering)
+ const syncApi = await RemoteSynchronizer.getSyncAPI(this.currency, this.host, this.port, this.server.conf.pair)
+ if (!syncApi.api) {
+ throw Error(DataErrors[DataErrors.CANNOT_CONNECT_TO_REMOTE_FOR_SYNC])
+ }
+ this.node = syncApi.api
+ this.peer = PeerDTO.fromJSONObject(syncApi.peering)
+ logger.info("Try with %s %s", this.peer.getURL(), this.peer.pubkey.substr(0, 6))
// We save this peer as a trusted peer for future contact
await this.server.PeeringService.submitP(DBPeer.fromPeerDTO(this.peer), false, false, true)
- logger.info("Try with %s %s", this.peer.getURL(), this.peer.pubkey.substr(0, 6))
- this.node = await connect(this.peer)
;(this.node as any).pubkey = this.peer.pubkey
- this.watcher.writeStatus('Connecting to ' + this.host + '...')
}
- async initWithKnownLocalAndToAndCurrency(to: number, localNumber: number, currency: string): Promise<void> {
+ private static async getSyncAPI(currency: string, host: string, port: number, keypair: Keypair) {
+ let api: IRemoteContacter|undefined
+ let peering: any
+ logger.info('Connecting to ' + host + '...')
+ try {
+ const contacter = await connect(PeerDTO.fromJSONObject({ endpoints: [`BASIC_MERKLED_API ${host} ${port}`]}), RemoteSynchronizer.contacterOptions.timeout)
+ peering = await contacter.getPeer()
+ api = new BMARemoteContacter(contacter)
+ } catch (e) {
+ logger.warn(`Node does not support BMA, trying WS2P...`)
+ }
+
+ // If BMA is unreachable, let's try WS2P
+ if (!api) {
+ const pair = KeyGen(keypair.pub, keypair.sec)
+ const connection = WS2PConnection.newConnectionToAddress(1,
+ `ws://${host}:${port}`,
+ new (class SyncMessageHandler implements WS2PMessageHandler {
+ async answerToRequest(json: any, c: WS2PConnection): Promise<WS2PResponse> {
+ throw Error(DataErrors[DataErrors.CANNOT_ARCHIVE_CHUNK_WRONG_SIZE])
+ }
+ async handlePushMessage(json: any, c: WS2PConnection): Promise<void> {
+ logger.warn('Receiving push messages, which are not allowed during a SYNC.', json)
+ }
+ }),
+ new WS2PPubkeyLocalAuth(currency, pair, '00000000'),
+ new WS2PPubkeyRemoteAuth(currency, pair)
+ )
+ const requester = WS2PRequester.fromConnection(connection)
+ peering = await requester.getPeer()
+ api = new WS2PRemoteContacter(requester)
+ }
+ if (!api) {
+ throw Error(DataErrors[DataErrors.CANNOT_CONNECT_TO_REMOTE_FOR_SYNC])
+ }
+ if (!peering) {
+ throw Error(DataErrors[DataErrors.NO_PEERING_AVAILABLE_FOR_SYNC])
+ }
+ if (peering.currency !== currency) {
+ throw Error(DataErrors[DataErrors.WRONG_CURRENCY_DETECTED])
+ }
+ return {
+ api,
+ peering
+ }
+ }
+
+ async initWithKnownLocalAndToAndCurrency(to: number, localNumber: number): Promise<void> {
this.to = to
this.localNumber = localNumber
- this.currency = currency
//=======
// Peers (just for P2P download)
//=======
let peers:(JSONDBPeer|null)[] = [];
if (!cliprogram.nopeers && (to - localNumber > 1000)) { // P2P download if more than 1000 blocs
this.watcher.writeStatus('Peers...');
- const merkle = await this.dal.merkleForPeers();
- const getPeers:(params:any) => Promise<HttpMerkleOfPeers> = this.node.getPeers.bind(this.node);
- const json2 = await getPeers({});
- const rm = new NodesMerkle(json2);
- if(rm.root() != merkle.root()){
- const leavesToAdd:string[] = [];
- const json = await getPeers({ leaves: true });
- json.leaves.forEach((leaf:string) => {
- if(merkle.leaves().indexOf(leaf) == -1){
- leavesToAdd.push(leaf);
- }
- });
- peers = await Promise.all(leavesToAdd.map(async (leaf) => {
- try {
- const json3 = await getPeers({ "leaf": leaf });
- const jsonEntry = json3.leaf.value;
- const endpoint = jsonEntry.endpoints[0];
- this.watcher.writeStatus('Peer ' + endpoint);
- return jsonEntry;
- } catch (e) {
- logger.warn("Could not get peer of leaf %s, continue...", leaf);
- return null;
- }
- }))
- }
- else {
- this.watcher.writeStatus('Peers already known');
- }
+ peers = await this.node.getPeers()
}
if (!peers.length) {
@@ -168,97 +200,21 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
return this.node.getBlock(number)
}
- static async test(host: string, port: number): Promise<BlockDTO> {
- const peering = await Contacter.fetchPeer(host, port, this.contacterOptions);
- const node = await connect(PeerDTO.fromJSONObject(peering));
- return node.getCurrent()
- }
-
- async syncPeers(fullSync: boolean, to?: number): Promise<void> {
- if (!cliprogram.nopeers && fullSync) {
-
- const peering = await Contacter.fetchPeer(this.host, this.port, RemoteSynchronizer.contacterOptions);
-
- let peer = PeerDTO.fromJSONObject(peering);
- logger.info("Try with %s %s", peer.getURL(), peer.pubkey.substr(0, 6));
- let node:any = await connect(peer);
- node.pubkey = peer.pubkey;
- logger.info('Sync started.');
-
- this.watcher.writeStatus('Peers...');
- await this.syncPeer(node);
- const merkle = await this.dal.merkleForPeers();
- const getPeers:(params:any) => Promise<HttpMerkleOfPeers> = node.getPeers.bind(node);
- const json2 = await getPeers({});
- const rm = new NodesMerkle(json2);
- if(rm.root() != merkle.root()){
- const leavesToAdd:string[] = [];
- const json = await getPeers({ leaves: true });
- json.leaves.forEach((leaf:string) => {
- if(merkle.leaves().indexOf(leaf) == -1){
- leavesToAdd.push(leaf);
- }
- });
- for (let i = 0; i < leavesToAdd.length; i++) {
- try {
- const leaf = leavesToAdd[i]
- const json3 = await getPeers({ "leaf": leaf });
- const jsonEntry = json3.leaf.value;
- const sign = json3.leaf.value.signature;
- const entry:any = {};
- entry.version = jsonEntry.version
- entry.currency = jsonEntry.currency
- entry.pubkey = jsonEntry.pubkey
- entry.endpoints = jsonEntry.endpoints
- entry.block = jsonEntry.block
- entry.signature = sign;
- this.watcher.writeStatus('Peer ' + entry.pubkey);
- this.watcher.peersPercent((i + 1) / leavesToAdd.length * 100)
- await this.PeeringService.submitP(entry, false, to === undefined);
- } catch (e) {
- logger.warn(e && e.message || e)
- }
- }
- this.watcher.peersPercent(100)
- }
- else {
- this.watcher.writeStatus('Peers already known');
- }
+ static async test(currency: string, host: string, port: number, keypair: Keypair): Promise<BlockDTO> {
+ const syncApi = await RemoteSynchronizer.getSyncAPI(currency, host, port, keypair)
+ const current = await syncApi.api.getCurrent()
+ if (!current) {
+ throw Error(DataErrors[DataErrors.REMOTE_HAS_NO_CURRENT_BLOCK])
}
+ return current
}
- //============
- // Peer
- //============
- private async syncPeer (node:any) {
-
- // Global sync vars
- const remotePeer = PeerDTO.fromJSONObject({});
- const json = await node.getPeer();
- remotePeer.version = json.version
- remotePeer.currency = json.currency
- remotePeer.pubkey = json.pub
- remotePeer.endpoints = json.endpoints
- remotePeer.blockstamp = json.block
- remotePeer.signature = json.signature
- const entry = remotePeer.getRawUnsigned();
- const signature = dos2unix(remotePeer.signature);
- // Parameters
- if(!(entry && signature)){
- throw 'Requires a peering entry + signature';
- }
-
- let remoteJsonPeer:any = json
- remoteJsonPeer.pubkey = json.pubkey;
- let signatureOK = this.PeeringService.checkPeerSignature(remoteJsonPeer);
- if (!signatureOK) {
- this.watcher.writeStatus('Wrong signature for peer #' + remoteJsonPeer.pubkey);
- }
- try {
- await this.PeeringService.submitP(remoteJsonPeer);
- } catch (err) {
- if (err.indexOf !== undefined && err.indexOf(CrawlerConstants.ERRORS.NEWER_PEER_DOCUMENT_AVAILABLE.uerr.message) !== -1 && err != CrawlerConstants.ERROR.PEER.UNKNOWN_REFERENCE_BLOCK) {
- throw err;
+ async syncPeers(fullSync: boolean, to?: number): Promise<void> {
+ const peers = await this.node.getPeers()
+ for (const p of peers) {
+ try {
+ await this.PeeringService.submitP(DBPeer.fromPeerDTO(PeerDTO.fromJSONObject(p)))
+ } catch (e) {
}
}
}
@@ -268,29 +224,3 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
await pullSandboxToLocalServer(this.currency, this.node, this.server, this.server.logger, this.watcher, 1, false)
}
}
-
-class NodesMerkle {
-
- private depth:number
- private nodesCount:number
- private leavesCount:number
- private merkleRoot:string
-
- constructor(json:any) {
- this.depth = json.depth
- this.nodesCount = json.nodesCount
- this.leavesCount = json.leavesCount
- this.merkleRoot = json.root;
- }
-
- // var i = 0;
- // this.levels = [];
- // while(json && json.levels[i]){
- // this.levels.push(json.levels[i]);
- // i++;
- // }
-
- root() {
- return this.merkleRoot
- }
-}
diff --git a/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts b/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts
new file mode 100644
index 0000000000000000000000000000000000000000..1a45c64f92c10c7d4b9476ecf7f282ceb6e7b3db
--- /dev/null
+++ b/app/modules/crawler/lib/sync/WS2PRemoteContacter.ts
@@ -0,0 +1,49 @@
+// Source file from duniter: Crypto-currency software to manage libre currency such as Äž1
+// Copyright (C) 2018 Cedric Moreau <cem.moreau@gmail.com>
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+import {NewLogger} from "../../../../lib/logger"
+import {IRemoteContacter} from "./IRemoteContacter";
+import {Contacter} from "../contacter";
+import {WS2PRequester} from "../../../ws2p/lib/WS2PRequester";
+import {DBPeer, JSONDBPeer} from "../../../../lib/db/DBPeer";
+import {BlockDTO} from "../../../../lib/dto/BlockDTO";
+import {PeerDTO} from "../../../../lib/dto/PeerDTO";
+import {HttpRequirements} from "../../../bma/lib/dtos";
+
+const logger = NewLogger()
+
+export class WS2PRemoteContacter implements IRemoteContacter {
+
+ getRequirementsPending(min: number): Promise<HttpRequirements> {
+ return this.requester.getRequirementsPending(min)
+ }
+
+ constructor(protected requester: WS2PRequester) {
+ }
+
+ getBlock(number: number): Promise<BlockDTO | null> {
+ return this.requester.getBlock(number)
+ }
+
+ getCurrent(): Promise<BlockDTO | null> {
+ return this.requester.getCurrent()
+ }
+
+ async getPeers(): Promise<(JSONDBPeer | null)[]> {
+ return (await this.requester.getPeers()).map(p => DBPeer.fromPeerDTO(PeerDTO.fromJSONObject(p)))
+ }
+
+ getName(): string {
+ return "WS2P remote"
+ }
+}
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index be9fa5851bc5d917a7a9efe632ad10daa511eae8..6c9e3687ea767ca0ad91d530dfa4f539e201fcf5 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -27,7 +27,6 @@ const nuuid = require('node-uuid');
const logger = require('../../../lib/logger').NewLogger('ws2p')
const MAXIMUM_ERRORS_COUNT = 5
-const REQUEST_TIMEOUT_VALUE = WS2PConstants.REQUEST_TIMEOUT
enum WS2P_ERR {
REJECTED_PUBKEY_OR_INCORRECT_ASK_SIGNATURE_FROM_REMOTE,
@@ -323,8 +322,8 @@ export class WS2PConnection {
connectionTimeout:number
requestTimeout:number
} = {
- connectionTimeout: REQUEST_TIMEOUT_VALUE,
- requestTimeout: REQUEST_TIMEOUT_VALUE
+ connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TIMEOUT
},
private expectedPub:string = "",
private expectedWS2PUID:string = ""
@@ -346,8 +345,8 @@ export class WS2PConnection {
connectionTimeout:number,
requestTimeout:number
} = {
- connectionTimeout: REQUEST_TIMEOUT_VALUE,
- requestTimeout: REQUEST_TIMEOUT_VALUE
+ connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TIMEOUT
},
expectedPub:string = "",
expectedWS2PUID:string = "") {
@@ -377,8 +376,8 @@ export class WS2PConnection {
connectionTimeout:number
requestTimeout:number
} = {
- connectionTimeout: REQUEST_TIMEOUT_VALUE,
- requestTimeout: REQUEST_TIMEOUT_VALUE
+ connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TIMEOUT
},
expectedPub:string = "") {
const onWsOpened = Promise.resolve()
diff --git a/app/modules/ws2p/lib/WS2PDocpoolPuller.ts b/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
index cc44dc48078102263157eb6c0e42ecd751db9657..a720d2455eacf9acd7e485bc1e00c4b7b9175d57 100644
--- a/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
+++ b/app/modules/ws2p/lib/WS2PDocpoolPuller.ts
@@ -29,7 +29,11 @@ export class WS2PDocpoolPuller {
return pullSandboxToLocalServer(this.server.conf.currency, {
getRequirementsPending: (minCert = 1) => {
return requester.getRequirementsPending(minCert)
- }
+ },
+ getName: () => this.connection.pubkey,
+ getPeers: async () => [],
+ getCurrent: async () => null,
+ getBlock: async () => null,
}, this.server, this.server.logger)
}
}
diff --git a/app/modules/ws2p/lib/WS2PRequester.ts b/app/modules/ws2p/lib/WS2PRequester.ts
index bfa2ea23589915a181d8488b74573b89e1f851e0..e043742eae4d1f2a82661e123992aec020eb9e70 100644
--- a/app/modules/ws2p/lib/WS2PRequester.ts
+++ b/app/modules/ws2p/lib/WS2PRequester.ts
@@ -13,6 +13,7 @@
import {WS2PConnection} from "./WS2PConnection"
import {BlockDTO} from "../../../lib/dto/BlockDTO"
+import {PeerDTO} from "../../../lib/dto/PeerDTO";
export enum WS2P_REQ {
KNOWN_PEERS,
@@ -32,6 +33,14 @@ export class WS2PRequester {
return new WS2PRequester(ws2pc)
}
+ getPeer(): Promise<PeerDTO> {
+ return this.query(WS2P_REQ.PEER_DOCUMENT)
+ }
+
+ getPeers(): Promise<PeerDTO[]> {
+ return this.query(WS2P_REQ.KNOWN_PEERS)
+ }
+
getCurrent(): Promise<BlockDTO> {
return this.query(WS2P_REQ.CURRENT)
}
diff --git a/test/integration/misc/cli.ts b/test/integration/misc/cli.ts
index a4540493bfc591fa046194e28e995b6327cccb0f..3f22278ecca404c1109dc55cec5214872fa61e6f 100644
--- a/test/integration/misc/cli.ts
+++ b/test/integration/misc/cli.ts
@@ -58,13 +58,13 @@ describe("CLI", function() {
/***************
* Normal nodes
*/
- return fakeSyncServer(onReadBlockchainChunk, onReadParticularBlock, onPeersRequested);
+ return fakeSyncServer('duniter_unit_test_currency', onReadBlockchainChunk, onReadParticularBlock, onPeersRequested);
} else if (index == 2) {
/***************
* Node with wrong chaining between 2 chunks of blocks
*/
- return fakeSyncServer((count:number, from:number) => {
+ return fakeSyncServer('duniter_unit_test_currency', (count:number, from:number) => {
// We just need to send the wrong chunk
from = from - count;
return Promise.resolve(blockchain.blocks.slice(from, from + count));
@@ -74,7 +74,7 @@ describe("CLI", function() {
/***************
* Node with wrong chaining between 2 blocks
*/
- return fakeSyncServer((count:number, from:number) => {
+ return fakeSyncServer('duniter_unit_test_currency', (count:number, from:number) => {
// We just need to send the wrong chunk
const chunk = blockchain.blocks.slice(from, from + count).map((block:any, index2:number) => {
if (index2 === 10) {
@@ -90,7 +90,7 @@ describe("CLI", function() {
/***************
* Node with apparent good chaining, but one of the hashs is WRONG
*/
- return fakeSyncServer((count:number, from:number) => {
+ return fakeSyncServer('duniter_unit_test_currency', (count:number, from:number) => {
// We just need to send the wrong chunk
const chunk = blockchain.blocks.slice(from, from + count).map((block:any, index2:number) => {
if (index2 === 10) {
@@ -109,6 +109,7 @@ describe("CLI", function() {
}))
farmOfServers.map((server, index) => {
const peer = {
+ currency: 'duniter_unit_test_currency',
endpoints: [['BASIC_MERKLED_API', server.host, server.port].join(' ')],
pubkey: hashf(index + ""),
hash: hashf(index + "").toUpperCase()
@@ -133,14 +134,14 @@ describe("CLI", function() {
it('sync 7 blocks (fast)', async () => {
await execute(['reset', 'data']);
- await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '7', '--nocautious', '--nointeractive', '--noshuffle']);
+ await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '7', 'duniter_unit_test_currency', '--nocautious', '--nointeractive', '--noshuffle']);
const res = await execute(['export-bc', '--nostdout']);
res[res.length - 1].should.have.property('number').equal(7);
res.should.have.length(7 + 1); // blocks #0..#7
})
it('sync 4 blocks (cautious)', async () => {
- await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '11', '--nointeractive']);
+ await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '11', 'duniter_unit_test_currency', '--nointeractive']);
const res = await execute(['export-bc', '--nostdout']);
res[res.length - 1].should.have.property('number').equal(11);
res.should.have.length(11 + 1);
@@ -153,7 +154,7 @@ describe("CLI", function() {
})
it('[spawn] sync 10 first blocks --memory', async () => {
- await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '10', '--memory', '--cautious', '--nointeractive']);
+ await execute(['sync', fakeServer.host + ':' + String(fakeServer.port), '10', 'duniter_unit_test_currency', '--memory', '--cautious', '--nointeractive']);
})
});
diff --git a/test/integration/tools/test-framework.ts b/test/integration/tools/test-framework.ts
index 52ced0319e67cefb91e669420afdfd896603b502..fe7d475d6532c3781403a94040bd9f79767dbef3 100644
--- a/test/integration/tools/test-framework.ts
+++ b/test/integration/tools/test-framework.ts
@@ -1,10 +1,15 @@
-import {catUser, NewTestingServer, tacUser, TestingServer} from "./toolbox"
+import {catUser, NewTestingServer, tacUser, TestingServer, tocUser} from "./toolbox"
import {TestUser} from "./TestUser"
import * as assert from 'assert'
-export function writeBasicTestWith2Users(writeTests: (test: (testTitle: string, fn: (server: TestingServer, cat: TestUser, tac: TestUser) => Promise<void>) => void) => void) {
+export function writeBasicTestWith2Users(writeTests: (
+ test: (
+ testTitle: string,
+ fn: (server: TestingServer, cat: TestUser, tac: TestUser, toc: TestUser) => Promise<void>
+ ) => void
+) => void) {
- let s1:TestingServer, cat:TestUser, tac:TestUser
+ let s1:TestingServer, cat:TestUser, tac:TestUser, toc:TestUser
before(async () => {
s1 = NewTestingServer({
@@ -16,12 +21,13 @@ export function writeBasicTestWith2Users(writeTests: (test: (testTitle: string,
})
cat = catUser(s1)
tac = tacUser(s1)
+ toc = tocUser(s1)
await s1.prepareForNetwork()
})
- writeTests((title, cb: (server: TestingServer, cat: TestUser, tac: TestUser) => Promise<void>) => {
+ writeTests((title, cb: (server: TestingServer, cat: TestUser, tac: TestUser, toc: TestUser) => Promise<void>) => {
it(title, async () => {
- await cb(s1, cat, tac)
+ await cb(s1, cat, tac, toc)
})
})
}
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index ad761b1c3303e0e7eaf251748c31ee562fab91c6..f6688e60dd7004535a26c6e790a55be673eee082 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -158,7 +158,7 @@ export const createUser = async (uid:string, pub:string, sec:string, defaultServ
return new TestUser(uid, keyring, { server: defaultServer });
}
-export const fakeSyncServer = async (readBlocksMethod:any, readParticularBlockMethod:any, onPeersRequested:any) => {
+export const fakeSyncServer = async (currency: string, readBlocksMethod:any, readParticularBlockMethod:any, onPeersRequested:any) => {
const host = HOST;
const port = PORT++;
@@ -181,6 +181,7 @@ export const fakeSyncServer = async (readBlocksMethod:any, readParticularBlockMe
// Mock BMA method for sync mocking
httpMethods.httpGET('/network/peering', async () => {
return {
+ currency,
endpoints: [['BASIC_MERKLED_API', host, port].join(' ')]
}
}, noLimit);
@@ -664,7 +665,7 @@ export class TestingServer {
}
}
- async enableWS2P(port: number = PORT++) {
+ async enableWS2P(port: number = PORT++): Promise<TestWS2PAPI> {
const cluster = WS2PCluster.plugOn(this._server)
await (this._server.ws2pCluster as WS2PCluster).listen(HOST, port)
const doConnection = (pair: Key, ws2pId: string, constructor: new (
@@ -686,11 +687,20 @@ export class TestingServer {
},
connectForSync: (pair: Key, ws2pId: string) => {
return doConnection(pair, ws2pId, WS2PPubkeySyncLocalAuth)
- }
+ },
+ host: HOST,
+ port
}
}
}
+export interface TestWS2PAPI {
+ connect: (pair: Key, ws2pId: string) => WS2PRequester
+ connectForSync: (pair: Key, ws2pId: string) => WS2PRequester
+ host: string
+ port: number
+}
+
export async function newWS2PBidirectionnalConnection(currency:string, k1:Key, k2:Key, serverHandler:WS2PMessageHandler) {
let i = 1
let port = PORT++
@@ -785,4 +795,13 @@ export function tacUser(server: TestingServer) {
{
server
})
+}
+
+export function tocUser(server: TestingServer) {
+ return new TestUser('toc', {
+ pub: 'DKpQPUL4ckzXYdnDRvCRKAm1gNvSdmAXnTrJZ7LvM5Qo',
+ sec: '64EYRvdPpTfLGGmaX5nijLXRqWXaVz8r1Z1GtaahXwVSJGQRn7tqkxLb288zwSYzELMEG5ZhXSBYSxsTsz1m9y8F'},
+ {
+ server
+ })
}
\ No newline at end of file
diff --git a/test/integration/ws2p/ws2p_sync.ts b/test/integration/ws2p/ws2p_sync.ts
index a1c812b8ea0efd76fd6fdc6d2040820f5bb0a889..b6852290e1dea562a8040c5fc2797b3d0a0a9451 100644
--- a/test/integration/ws2p/ws2p_sync.ts
+++ b/test/integration/ws2p/ws2p_sync.ts
@@ -13,21 +13,48 @@
import {WS2PConstants} from "../../../app/modules/ws2p/lib/constants"
import {assertEqual, assertNotNull, createCurrencyWith2Blocks, writeBasicTestWith2Users} from "../tools/test-framework"
+import {NewTestingServer, TestWS2PAPI} from "../tools/toolbox";
+import {assertThrows} from "../../unit-tools";
+import {CrawlerDependency} from "../../../app/modules/crawler/index";
describe('WS2P sync', () => writeBasicTestWith2Users((test) => {
- WS2PConstants.CONNEXION_TIMEOUT = 100
- WS2PConstants.REQUEST_TIMEOUT= 100
+
+ // We want the test to fail quickly
+ WS2PConstants.CONNEXION_TIMEOUT = 1000
+ WS2PConstants.REQUEST_TIMEOUT = 1000
+
+ let ws2p: TestWS2PAPI
test('should be able to init with 2 blocks', async (s1, cat, tac) => {
await createCurrencyWith2Blocks(s1, cat, tac)
})
- test('if we disable the changes API', async (s1, cat, tac) => {
- const ws2p = await s1.enableWS2P()
- const ws = (await ws2p).connectForSync(tac.keypair, '12345678')
+ test('we should be able to connect for SYNC', async (s1, cat, tac) => {
+ ws2p = await s1.enableWS2P()
+ const ws = ws2p.connectForSync(tac.keypair, '12345678')
const current = await ws.getCurrent()
assertNotNull(current)
assertEqual(2, current.number)
})
+
+ test('we should NOT be able to reconnect for SYNC', async (s1, cat, tac) => {
+ const ws = ws2p.connectForSync(tac.keypair, '22222222')
+ await assertThrows(ws.getCurrent(), 'WS2P connection timeout')
+ })
+
+ test('we should be able to connect for SYNC with toc', async (s1, cat, tac, toc) => {
+ const ws = ws2p.connectForSync(toc.keypair, '33333333')
+ const current = await ws.getCurrent()
+ assertNotNull(current)
+ assertEqual(2, current.number)
+ })
+
+ test('we should be able to make a full sync with cat', async (s1, cat, tac, toc) => {
+ const s2 = NewTestingServer({ pair: cat.keypair })
+ await s2.initWithDAL()
+ // We sync on s1
+ await CrawlerDependency.duniter.methods.synchronize(s1.conf.currency, s2._server, ws2p.host, ws2p.port, 2, 250).syncPromise
+ assertNotNull(await s2.dal.getCurrentBlockOrNull())
+ })
}))