From 7b79c56f714ff9baacf860cbe7cccbff68bfe060 Mon Sep 17 00:00:00 2001
From: Benoit Lavenier <benoit.lavenier@e-is.pro>
Date: Fri, 2 Jun 2023 17:39:13 +0200
Subject: [PATCH] fix(1445): When synchronize, use BMA/BMAS peers having a path
(e.g. /bma): PeerDTP.getURL() was not using the path - close #1445
---
app/lib/common-libs/constants.ts | 2 +-
app/lib/dto/PeerDTO.ts | 8 +++-
app/modules/bma/index.ts | 26 ++++++-------
app/modules/crawler/index.ts | 12 +++---
app/modules/crawler/lib/contacter.ts | 2 +-
.../crawler/lib/sync/RemoteSynchronizer.ts | 10 ++---
.../crawler/lib/sync/p2p/p2p-candidate.ts | 1 +
.../crawler/lib/sync/v2/GlobalIndexStream.ts | 37 +++++++++++++------
app/modules/prover/index.ts | 2 +-
app/service/PeeringService.ts | 7 ++--
10 files changed, 64 insertions(+), 43 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 0cf4179ea..91266e0aa 100755
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -54,7 +54,7 @@ const CONDITIONS =
CSV_INTEGER +
"\\))))*";
const CONDITION_SIG_PUBKEY = "SIG\\((" + PUBKEY + ")\\)";
-const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
+const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/;
const BMAS_REGEXP = /^BMAS( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/;
const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
const WS2P_REGEXP = /^WS2P (?:[1-9][0-9]* )?([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/;
diff --git a/app/lib/dto/PeerDTO.ts b/app/lib/dto/PeerDTO.ts
index 50c9b109c..62e70afdf 100644
--- a/app/lib/dto/PeerDTO.ts
+++ b/app/lib/dto/PeerDTO.ts
@@ -113,6 +113,7 @@ export class PeerDTO implements Cloneable {
ipv4: matchesBMA[4] || "",
ipv6: matchesBMA[6] || "",
port: parseInt(matchesBMA[8]) || 9101,
+ path: matchesBMA[10] || "",
};
} else if (matchesBMAS) {
notFound = false;
@@ -271,7 +272,8 @@ export class PeerDTO implements Cloneable {
getURL() {
const bma = this.getBMA();
let base = this.getHostPreferDNS();
- if (bma.port) base += ":" + bma.port;
+ if (base && bma.port) base += ":" + bma.port;
+ if (base && bma.path) base += bma.path;
return base;
}
@@ -377,4 +379,8 @@ export class PeerDTO implements Cloneable {
}
return 0;
}
+
+ static isBMA(endpoint: string) {
+ return endpoint && !!endpoint.match(/^(BASIC_MERKLED_API|BMAS)/) || false;
+ }
}
diff --git a/app/modules/bma/index.ts b/app/modules/bma/index.ts
index ca769502a..73f4f5d5f 100644
--- a/app/modules/bma/index.ts
+++ b/app/modules/bma/index.ts
@@ -212,8 +212,8 @@ export const BmaDependency = {
}
}
if (!conf.nobma) {
- server.addEndpointsDefinitions(() =>
- Promise.resolve(getEndpoint(conf))
+ server.addEndpointsDefinitions(async () =>
+ getEndpoint(conf)
);
server.addWrongEndpointFilter((endpoints: string[]) =>
getWrongEndpoints(endpoints, server.conf.pair.pub)
@@ -241,23 +241,23 @@ export const BmaDependency = {
async function getWrongEndpoints(endpoints: string[], selfPubkey: string) {
const wrongs: string[] = [];
await Promise.all(
- endpoints.map(async (theEndpoint: string) => {
- let remote = PeerDTO.endpoint2host(theEndpoint);
- try {
- // We test only BMA APIs, because other may exist and we cannot judge against them
- if (theEndpoint.startsWith("BASIC_MERKLED_API")) {
- let answer = await rp("http://" + remote + "/network/peering", {
+ endpoints
+ .filter(PeerDTO.isBMA) // We test only BMA APIs, because other may exist and we cannot judge against them
+ .map(async (ep: string) => {
+ const peer = PeerDTO.fromJSONObject({ endpoints: [ep] });
+ try {
+ const protocol = ep.startsWith("BMAS") || peer.getPort() == 443 ? "https" : "http";
+ const answer = await rp(protocol + "://" + peer.getURL() + "/network/peering", {
json: true,
});
if (!answer || answer.pubkey != selfPubkey) {
throw Error("Not same pubkey as local instance");
}
+ } catch (e) {
+ wrongs.push(ep);
}
- } catch (e) {
- wrongs.push(theEndpoint);
- }
- })
- );
+ })
+ );
return wrongs;
}
diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts
index 5c4bf9768..7257de8c4 100644
--- a/app/modules/crawler/index.ts
+++ b/app/modules/crawler/index.ts
@@ -309,7 +309,7 @@ export const CrawlerDependency = {
? [
{
endpoints: [
- ["BASIC_MERKLED_API", fromHost, fromPort].join(" "),
+ [fromPort == "443" ? "BMAS" : "BASIC_MERKLED_API", fromHost, fromPort].join(" "),
],
},
]
@@ -358,7 +358,7 @@ export const CrawlerDependency = {
const { host, port } = extractHostPort(from);
try {
const peer = PeerDTO.fromJSONObject({
- endpoints: [["BASIC_MERKLED_API", host, port].join(" ")],
+ endpoints: [[port == "443" ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ")],
});
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
@@ -405,7 +405,7 @@ export const CrawlerDependency = {
const { host, port } = extractHostPort(from);
try {
const peer = PeerDTO.fromJSONObject({
- endpoints: [["BASIC_MERKLED_API", host, port].join(" ")],
+ endpoints: [[port == "443" ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ")],
});
const fromHost = peer.getHostPreferDNS();
const fromPort = peer.getPort();
@@ -459,7 +459,7 @@ export const CrawlerDependency = {
const { host: toHost, port: toPort } = extractHostPort(target);
try {
const peer = PeerDTO.fromJSONObject({
- endpoints: [["BASIC_MERKLED_API", host, port].join(" ")],
+ endpoints: [[port == "443" ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ")],
});
logger.info("Looking at %s...", source);
try {
@@ -508,7 +508,7 @@ export const CrawlerDependency = {
const { host, port } = extractHostPort(source);
try {
const peer = PeerDTO.fromJSONObject({
- endpoints: [["BASIC_MERKLED_API", host, port].join(" ")],
+ endpoints: [[port == "443" ? "BMAS" : "BASIC_MERKLED_API"].join(" ")],
});
logger.info("Looking at %s...", source);
try {
@@ -752,7 +752,7 @@ export const CrawlerDependency = {
? [
{
endpoints: [
- ["BASIC_MERKLED_API", fromHost, fromPort].join(" "),
+ [fromPort == "443" ? "BMAS" : "BASIC_MERKLED_API", fromHost, fromPort].join(" "),
],
},
]
diff --git a/app/modules/crawler/lib/contacter.ts b/app/modules/crawler/lib/contacter.ts
index 8bac05189..59268a987 100644
--- a/app/modules/crawler/lib/contacter.ts
+++ b/app/modules/crawler/lib/contacter.ts
@@ -19,9 +19,9 @@ const sanitize = require("../../../modules/bma/lib/sanitize");
const dtos = require("../../../modules/bma").BmaDependency.duniter.methods.dtos;
export class Contacter {
- path: string = "";
options: { timeout: number };
fullyQualifiedHost: string;
+ path: string = "";
constructor(
public readonly host: string,
diff --git a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
index 7c31a61d5..fa3c1099d 100644
--- a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
@@ -158,16 +158,16 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
const contacter = await connect(
PeerDTO.fromJSONObject({
endpoints: [
- `BASIC_MERKLED_API ${host} ${port}` +
- ((path && " " + path) || ""),
+ [port == 443 ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ") +
+ (path ? (' ' + path) : '')
],
}),
3000
);
peering = await contacter.getPeer();
api = new BMARemoteContacter(contacter);
- endpoint =
- `BASIC_MERKLED_API ${host} ${port}` + ((path && " " + path) || "");
+ endpoint = [port == 443 ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ") +
+ (path ? (' ' + path) : '');
} catch (e) {}
}
@@ -176,7 +176,7 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
const pair = new Key(keypair.pub, keypair.sec);
const connection = WS2PConnection.newConnectionToAddress(
1,
- `ws://${host}:${port}${path || ""}`,
+ `ws://${host}:${port}${path || ''}`,
new (class SyncMessageHandler implements WS2PMessageHandler {
async answerToRequest(
json: any,
diff --git a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
index 8f81ed886..56fca9b04 100644
--- a/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
+++ b/app/modules/crawler/lib/sync/p2p/p2p-candidate.ts
@@ -121,6 +121,7 @@ export class P2pCandidate {
isBMA: true,
port: bmaAPI.port,
host: bmaHost,
+ path: bmaAPI.path,
});
}
if (ws2pAPI) {
diff --git a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
index 694e52069..9abc78f27 100644
--- a/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
+++ b/app/modules/crawler/lib/sync/v2/GlobalIndexStream.ts
@@ -39,6 +39,7 @@ let sync_mindex: any[] = [];
let sync_cindex: any[] = [];
let sync_nextExpiring = 0;
let sync_bindexSize = 0;
+let sync_txs: any[] = [];
let txCount = 0;
let logger = NewLogger();
@@ -89,7 +90,6 @@ export class GlobalIndexStream extends Duplex {
private numberOfChunksToDownload: number;
private memToCopyDone = false;
- private mapInjection: { [k: string]: any } = {};
constructor(
private conf: ConfDTO,
@@ -438,16 +438,31 @@ export class GlobalIndexStream extends Duplex {
})
);
- if (this.conf.storage && this.conf.storage.transactions) {
- await Promise.all(
- blocks.map((block) =>
- this.dal.saveTxsInFiles(
- block.transactions,
- block.number,
- block.medianTime
- )
- )
- );
+ if (this.conf.storage?.transactions) {
+ // if cautious, use a save (insert or update)
+ if (this.cautious) {
+ await Promise.all(
+ blocks.map((block) =>
+ this.dal.saveTxsInFiles(
+ block.transactions,
+ block.number,
+ block.medianTime
+ )
+ )
+ );
+ }
+ // If not cautious: use insert only
+ else {
+ await Promise.all(
+ blocks.map((block) =>
+ this.dal.insertTxsInFiles(
+ block.transactions,
+ block.number,
+ block.medianTime
+ )
+ )
+ );
+ }
}
logger.debug("Total tx count: %s", txCount);
diff --git a/app/modules/prover/index.ts b/app/modules/prover/index.ts
index 8bdca8b55..2d4a3d1dd 100644
--- a/app/modules/prover/index.ts
+++ b/app/modules/prover/index.ts
@@ -351,7 +351,7 @@ function proveAndSend(
next();
} else {
const peer = PeerDTO.fromJSONObject({
- endpoints: [["BASIC_MERKLED_API", host, port].join(" ")],
+ endpoints: [[port == "443" ? "BMAS" : "BASIC_MERKLED_API", host, port].join(" ")],
});
program.show && console.log(proven.getRawSigned());
logger.info("Posted block " + proven.getRawSigned());
diff --git a/app/service/PeeringService.ts b/app/service/PeeringService.ts
index 6fd0a104e..aa8a7ca60 100755
--- a/app/service/PeeringService.ts
+++ b/app/service/PeeringService.ts
@@ -79,7 +79,7 @@ export class PeeringService {
return this.getOtherEndpoints(
localPeer.endpoints,
localEndpoints
- ).filter((ep) => ep.match(/^BASIC_MERKLED_API/));
+ ).filter(PeerDTO.isBMA);
}
checkPeerSignature(p: PeerDTO) {
@@ -363,9 +363,8 @@ export class PeeringService {
const ws2pAccess = PeerDTO.fromJSONObject(p2).getFirstNonTorWS2P();
if (ws2pAccess) {
logger.info(
- `WS2P access: ${ws2pAccess.host} :${ws2pAccess.port}${
- (ws2pAccess.path && " " + ws2pAccess.path) || ""
- }`
+ `WS2P access: ${ws2pAccess.host}:${ws2pAccess.port}`
+ + (ws2pAccess.path ? (" " + ws2pAccess.path) : "")
);
}
logger.debug(
--
GitLab