[enh] Add `sync-mempool-fwd` command (8cbeb23b) · Commits · nodes / typescript / duniter

app/lib/common-libs/constants.ts

+2 −0

Original line number	Diff line number	Diff line
		@@ -44,6 +44,7 @@ const UNLOCK = "(SIG\$" + INTEGER + "\$\|XHX\$" + XUNLOCK + "\$)"
		const CONDITIONS = "(&&\|\\\|\\\|\| \|[()]\|(SIG\$" + PUBKEY + "\$\|(XHX\$[A-F0-9]{64}\$\|CLTV\$" + CLTV_INTEGER + "\$\|CSV\$" + CSV_INTEGER + "\$)))*"

		const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
		const BMAS_REGEXP = /^BMAS( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/
		const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
		const WS2P_REGEXP = /^WS2P (?:[1-9][0-9]* )?([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*\|[0-9.]+\|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
		const WS2P_V2_REGEXP = /^WS2P ([1-9][0-9]) ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]\|[0-9.]+\|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
		@@ -110,6 +111,7 @@ export const CommonConstants = {
		SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS: 3,

		BMA_REGEXP,
		BMAS_REGEXP,
		BMATOR_REGEXP,
		WS2P_REGEXP,
		WS2P_V2_REGEXP,

app/lib/dto/PeerDTO.ts

+23 −7

Original line number	Diff line number	Diff line
		@@ -94,17 +94,28 @@ export class PeerDTO implements Cloneable {
		}

		getBMA() {
		let bma: { dns?: string, ipv4?: string, ipv6?: string, port?: number } = {}
		let bma: { dns?: string, ipv4?: string, ipv6?: string, port?: number, path?: string } = {}
		let notFound = true
		this.endpoints.forEach((ep) => {
		const matches = notFound && ep.match(CommonConstants.BMA_REGEXP);
		if (matches) {
		const matchesBMA = notFound && ep.match(CommonConstants.BMA_REGEXP);
		const matchesBMAS = notFound && ep.match(CommonConstants.BMAS_REGEXP);
		if (matchesBMA) {
		notFound = false
		bma = {
		"dns": matches[2] \|\| '',
		"ipv4": matches[4] \|\| '',
		"ipv6": matches[6] \|\| '',
		"port": parseInt(matches[8]) \|\| 9101
		"dns": matchesBMA[2] \|\| '',
		"ipv4": matchesBMA[4] \|\| '',
		"ipv6": matchesBMA[6] \|\| '',
		"port": parseInt(matchesBMA[8]) \|\| 9101
		};
		}
		else if (matchesBMAS) {
		notFound = false
		bma = {
		"dns": matchesBMAS[2] \|\| '',
		"ipv4": matchesBMAS[4] \|\| '',
		"ipv6": matchesBMAS[6] \|\| '',
		"port": parseInt(matchesBMAS[8]) \|\| 9101,
		"path": matchesBMAS[10] \|\| ''
		};
		}
		});
		@@ -207,6 +218,11 @@ export class PeerDTO implements Cloneable {
		return bma.port ? bma.port : null;
		}

		getPath() {
		const bma = this.getBMA();
		return bma.path ? bma.path : null;
		}

		getHostPreferDNS() {
		const bma = this.getBMA();
		return (bma.dns ? bma.dns :

app/modules/crawler/index.ts

+36 −3

Original line number	Diff line number	Diff line
		@@ -33,9 +33,9 @@ import {CrawlerConstants} from "./lib/constants"
		import {ExitCodes} from "../../lib/common-libs/exit-codes"
		import {connect} from "./lib/connect"
		import {BMARemoteContacter} from "./lib/sync/BMARemoteContacter"
		import {applyMempoolRequirements, pullSandboxToLocalServer} from "./lib/sandbox"
		import {applyMempoolRequirements, forwardToServer, pullSandboxToLocalServer} from "./lib/sandbox"

		const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?$/
		const HOST_PATTERN = /^[^:/]+(:[0-9]{1,5})?(\/.*)?$/
		const FILE_PATTERN = /^(\/.+)$/

		export const CrawlerDependency = {
		@@ -281,6 +281,39 @@ export const CrawlerDependency = {
		}
		}
		}, {
		name: 'sync-mempool-fwd <from> <to> <search>',
		desc: 'Import all pending data from matching <search>',
		onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
		const source: string = params[0]
		const target: string = params[1]
		const search: string = params[2]
		if (!source \|\| !(source.match(HOST_PATTERN) \|\| source.match(FILE_PATTERN))) {
		throw 'Source of sync is required. (host[:port])'
		}
		if (!target \|\| !(target.match(HOST_PATTERN) \|\| target.match(FILE_PATTERN))) {
		throw 'Target of sync is required. (host[:port])'
		}
		const logger = NewLogger()
		const { host, port } = extractHostPort(source)
		const { host: toHost, port: toPort } = extractHostPort(target)
		try {
		const peer = PeerDTO.fromJSONObject({ endpoints: [['BASIC_MERKLED_API', host, port].join(' ')] })
		logger.info('Looking at %s...', source)
		try {
		const fromHost = await connect(peer)
		const res = await fromHost.getRequirements(search)
		await forwardToServer(server.conf.currency, res, toHost, toPort, logger)
		} catch (e) {
		logger.error(e);
		}

		await server.disconnect();
		} catch(e) {
		logger.error(e);
		throw Error("Exiting");
		}
		}
		}, {
		name: 'forward <number> <fromHost> <fromPort> <toHost> <toPort>',
		desc: 'Forward existing block <number> from a host to another',
		onDatabaseExecute: async (server:Server, conf:ConfDTO, program:any, params:any) => {
		@@ -474,7 +507,7 @@ export const CrawlerDependency = {
		function extractHostPort(source: string) {
		const sp = source.split(':')
		const onHost = sp[0]
		const onPort = parseInt(sp[1] ? sp[1] : '443') // Defaults to 443
		const onPort = sp[1] ? sp[1] : '443' // Defaults to 443
		return {
		host: onHost,
		port: onPort,

app/modules/crawler/lib/connect.ts

+5 −1

Original line number	Diff line number	Diff line
		@@ -18,7 +18,11 @@ import {PeerDTO} from "../../../lib/dto/PeerDTO";
		const DEFAULT_HOST = 'localhost';

		export const connect = (peer:PeerDTO, timeout:number\|null = null) => {
		return Promise.resolve(new Contacter(peer.getDns() \|\| peer.getIPv4() \|\| peer.getIPv6() \|\| DEFAULT_HOST, peer.getPort() as number, {
		return Promise.resolve(Contacter.fromHostPortPath(
		peer.getDns() \|\| peer.getIPv4() \|\| peer.getIPv6() \|\| DEFAULT_HOST,
		peer.getPort() as number,
		peer.getPath() as string,
		{
		timeout: timeout \|\| CrawlerConstants.DEFAULT_TIMEOUT
		}))
		}

app/modules/crawler/lib/contacter.ts

+13 −3

Original line number	Diff line number	Diff line
		@@ -12,6 +12,7 @@
		// GNU Affero General Public License for more details.

		import {CrawlerConstants} from "./constants"
		import {HttpMembershipList} from "../../bma/lib/dtos"

		const rp = require('request-promise');
		const sanitize = require('../../../modules/bma/lib/sanitize')
		@@ -19,6 +20,7 @@ const dtos = require('../../../modules/bma').BmaDependency.duniter.methods.dtos;

		export class Contacter {

		path: string = ''
		options:{ timeout:number }
		fullyQualifiedHost:string

		@@ -30,6 +32,12 @@ export class Contacter {
		this.fullyQualifiedHost = [host, port].join(':');
		}

		public static fromHostPortPath(host:string, port:number, path:string, opts: { timeout?: number }) {
		const contacter = new Contacter(host, port, opts)
		contacter.path = path
		return contacter
		}

		getSummary() {
		return this.get('/node/summary/', dtos.Summary)
		}
		@@ -106,7 +114,7 @@ export class Contacter {
		return this.post('/wot/revoke', dtos.Identity, { revocation: rev })
		}

		wotPending() {
		wotPending(): Promise<HttpMembershipList> {
		return this.get('/wot/pending', dtos.MembershipList)
		}

		@@ -128,8 +136,9 @@ export class Contacter {
		param = '?' + Object.keys(param).map((k) => [k, param[k]].join('=')).join('&');
		}
		try {
		const path = this.path \|\| ''
		const json = await rp.get({
		url: Contacter.protocol(this.port) + this.fullyQualifiedHost + url + (param !== undefined ? param : ''),
		url: Contacter.protocol(this.port) + this.fullyQualifiedHost + path + url + (param !== undefined ? param : ''),
		json: true,
		timeout: this.options.timeout
		});
		@@ -142,8 +151,9 @@ export class Contacter {

		private async post(url:string, dtoContract:any, data:any) {
		try {
		const path = this.path \|\| ''
		const json = await rp.post({
		url: Contacter.protocol(this.port) + this.fullyQualifiedHost + url,
		url: Contacter.protocol(this.port) + this.fullyQualifiedHost + path + url,
		body: data,
		json: true,
		timeout: this.options.timeout