From a94d5e414eec67fd89b7fdb394a6a312258727df Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 04:43:59 +0100
Subject: [PATCH 01/34] add dependence socks-proxy-agent
---
package.json | 1 +
1 file changed, 1 insertion(+)
diff --git a/package.json b/package.json
index 524a17cfa..97649b314 100644
--- a/package.json
+++ b/package.json
@@ -92,6 +92,7 @@
"scryptb": "6.0.4",
"seedrandom": "^2.4.3",
"sha1": "1.1.1",
+ "socks-proxy-agent": "^3.0.1",
"sqlite3": "3.1.4",
"tail": "^1.2.1",
"tweetnacl": "0.14.3",
--
GitLab
From 005a8c2db82727021dc520150c23a086a2c04d06 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 04:44:47 +0100
Subject: [PATCH 02/34] [enh] add Proxy type
---
app/lib/proxy.ts | 80 ++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 80 insertions(+)
create mode 100644 app/lib/proxy.ts
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
new file mode 100644
index 000000000..cdcb039dd
--- /dev/null
+++ b/app/lib/proxy.ts
@@ -0,0 +1,80 @@
+const SocksProxyAgent = require('socks-proxy-agent');
+
+const constants = require('./constants');
+const WS2PConstants = require('../modules/ws2p/lib/constants');
+
+export interface Proxies {
+ proxySocks: Proxy|undefined,
+ proxyTor: Proxy|undefined
+}
+
+export interface ProxyConf {
+ proxySocksAddress: string|undefined,
+ proxyTorAddress: string|undefined,
+ alwaysUseTor: boolean|undefined,
+ proxies: Proxies|undefined
+}
+
+export class Proxy {
+ private agent: any
+
+ constructor(proxy:string, type:string = "socks") {
+ if (type === "socks") {
+ this.agent = SocksProxyAgent("socks://"+proxy)
+ }
+ else {
+ this.agent = undefined
+ }
+ }
+
+ getAgent() {
+ return this.agent;
+ }
+
+ static defaultConf():ProxyConf {
+ return {
+ proxySocksAddress: undefined,
+ proxyTorAddress: undefined,
+ alwaysUseTor: undefined,
+ proxies: undefined
+ }
+ }
+
+ static createProxies(proxyConf: ProxyConf|undefined) : Proxies|undefined
+ {
+ if (proxyConf !== undefined) {
+ return {
+ proxySocks: (proxyConf.proxySocksAddress !== undefined) ? new Proxy(proxyConf.proxySocksAddress, "socks"):undefined,
+ proxyTor: (proxyConf.proxyTorAddress !== undefined) ? new Proxy(proxyConf.proxyTorAddress, "socks"):undefined
+ }
+ } else {
+ return undefined
+ }
+ }
+
+ static httpProxy(url:string, proxyConf: ProxyConf|undefined) {
+ return Proxy.chooseProxy(url, proxyConf, constants.ONION_ENDPOINT_REGEX)
+ }
+
+ static wsProxy(address:string, proxyConf: ProxyConf|undefined) {
+ return Proxy.chooseProxy(address, proxyConf, WS2PConstants.ONION_ENDPOINT_REGEX)
+ }
+
+ private static chooseProxy(address:string, proxyConf: ProxyConf|undefined, onionRegex:RegExp): Proxy|undefined {
+ if (proxyConf !== undefined) {
+ if (proxyConf.proxies === undefined) {
+ proxyConf.proxies = Proxy.createProxies(proxyConf)
+ }
+ if (proxyConf.proxies !== undefined) {
+ if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.getAgent() !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)))
+ {
+ return proxyConf.proxies.proxyTor
+ }
+ else if (proxyConf.proxies.proxySocks !== undefined && proxyConf.proxies.proxySocks.getAgent() !== undefined) {
+ return proxyConf.proxies.proxyTor
+ }
+ }
+ }
+ return undefined
+ }
+}
--
GitLab
From cbcfd1b694b8e9763ebfed08681b965da08ee9cf Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 04:47:01 +0100
Subject: [PATCH 03/34] [enh] add cli options for proxies conf
---
app/cli.ts | 5 +++++
index.ts | 43 +++++++++++++++++++++++++------------------
server.ts | 5 ++++-
3 files changed, 34 insertions(+), 19 deletions(-)
diff --git a/app/cli.ts b/app/cli.ts
index cedf3a7fa..bab8e0d65 100644
--- a/app/cli.ts
+++ b/app/cli.ts
@@ -49,6 +49,11 @@ export const ExecuteCommand = () => {
.option('--nostdout', 'Disable stdout printing for `export-bc` command')
.option('--noshuffle', 'Disable peers shuffling for `sync` command')
+ .option('--proxy-socks <host:port>', 'Use Socks Proxy')
+ .option('--proxy-tor <host:port>', 'Use Tor Socks Proxy')
+ .option('--tor-always', 'Pass all outgoing requests through the tor network')
+ .option('--tor-mixed', 'Pass only ".onion" outgoing requests through the tor network. It\'s the default behavior')
+
.option('--timeout <milliseconds>', 'Timeout to use when contacting peers', parseInt)
.option('--httplogs', 'Enable HTTP logs')
.option('--nohttplogs', 'Disable HTTP logs')
diff --git a/index.ts b/index.ts
index 3c6b04bc5..b1d842581 100644
--- a/index.ts
+++ b/index.ts
@@ -8,6 +8,7 @@ import {CrawlerDependency} from "./app/modules/crawler/index"
import {BmaDependency} from "./app/modules/bma/index"
import {WS2PDependency} from "./app/modules/ws2p/index"
import {Constants} from "./app/modules/prover/lib/constants"
+import { Proxy } from './app/lib/proxy';
const path = require('path');
const _ = require('underscore');
@@ -437,10 +438,9 @@ class Stack {
}
}
-function commandLineConf(program:any, conf:any = {}) {
+function commandLineConf(program:any, conf:ConfDTO = ConfDTO.mock()) {
conf = conf || {};
- conf.sync = conf.sync || {};
const cli = {
currency: program.currency,
cpu: program.cpu,
@@ -448,10 +448,11 @@ function commandLineConf(program:any, conf:any = {}) {
server: {
port: program.port,
},
- db: {
- mport: program.mport,
- mdb: program.mdb,
- home: program.home
+ proxies: {
+ proxySocks: program.proxySocks,
+ proxyTor: program.proxyTor,
+ torAlways: program.torAlways,
+ torMixed: program.torMixed
},
logs: {
http: program.httplogs,
@@ -465,19 +466,25 @@ function commandLineConf(program:any, conf:any = {}) {
timeout: program.timeout
};
+ // Declare proxyConf
+ if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.torAlways || cli.proxies.torMixed) {
+ conf.proxyConf = Proxy.defaultConf()
+ }
+
// Update conf
- if (cli.currency) conf.currency = cli.currency;
- if (cli.server.port) conf.port = cli.server.port;
- if (cli.cpu) conf.cpu = Math.max(0.01, Math.min(1.0, cli.cpu));
- if (cli.prefix) conf.prefix = Math.max(Constants.MIN_PEER_ID, Math.min(Constants.MAX_PEER_ID, cli.prefix));
- if (cli.logs.http) conf.httplogs = true;
- if (cli.logs.nohttp) conf.httplogs = false;
- if (cli.db.mport) conf.mport = cli.db.mport;
- if (cli.db.home) conf.home = cli.db.home;
- if (cli.db.mdb) conf.mdb = cli.db.mdb;
- if (cli.isolate) conf.isolate = cli.isolate;
- if (cli.timeout) conf.timeout = cli.timeout;
- if (cli.forksize != null) conf.forksize = cli.forksize;
+ if (cli.currency) conf.currency = cli.currency;
+ if (cli.server.port) conf.port = cli.server.port;
+ if (cli.cpu) conf.cpu = Math.max(0.01, Math.min(1.0, cli.cpu));
+ if (cli.prefix) conf.prefix = Math.max(Constants.MIN_PEER_ID, Math.min(Constants.MAX_PEER_ID, cli.prefix));
+ if (cli.proxies.proxySocks && conf.proxyConf) conf.proxyConf.proxySocksAddress = cli.proxies.proxySocks;
+ if (cli.proxies.proxyTor && conf.proxyConf) conf.proxyConf.proxyTorAddress = cli.proxies.proxyTor;
+ if (cli.proxies.torAlways && conf.proxyConf) conf.proxyConf.alwaysUseTor = true;
+ if (cli.proxies.torMixed && conf.proxyConf) conf.proxyConf.alwaysUseTor = false;
+ if (cli.logs.http) conf.httplogs = true;
+ if (cli.logs.nohttp) conf.httplogs = false;
+ if (cli.isolate) conf.isolate = cli.isolate;
+ if (cli.timeout) conf.timeout = cli.timeout;
+ if (cli.forksize != null) conf.forksize = cli.forksize;
return conf;
}
diff --git a/server.ts b/server.ts
index 2f7fbc5c1..f91364b26 100644
--- a/server.ts
+++ b/server.ts
@@ -24,6 +24,7 @@ import {PeerDTO} from "./app/lib/dto/PeerDTO"
import {OtherConstants} from "./app/lib/other_constants"
import {WS2PCluster} from "./app/modules/ws2p/lib/WS2PCluster"
import {DBBlock} from "./app/lib/db/DBBlock"
+import { Proxy } from './app/lib/proxy';
export interface HookableServer {
generatorGetJoinData: (...args:any[]) => Promise<any>
@@ -72,7 +73,7 @@ export class Server extends stream.Duplex implements HookableServer {
TransactionsService:TransactionService
private documentFIFO:GlobalFifoPromise
- constructor(home:string, memoryOnly:boolean, private overrideConf:any) {
+ constructor(home:string, memoryOnly:boolean, private overrideConf:ConfDTO) {
super({ objectMode: true })
this.home = home;
@@ -148,6 +149,8 @@ export class Server extends stream.Duplex implements HookableServer {
logger.debug('Loading conf...');
this.conf = await this.dal.loadConf(this.overrideConf, useDefaultConf)
// Default values
+ this.conf.proxyConf = this.conf.proxyConf === undefined ? Proxy.defaultConf() : this.conf.proxyConf
+ this.conf.proxyConf.alwaysUseTor = this.conf.proxyConf.alwaysUseTor === undefined ? false : this.conf.proxyConf.alwaysUseTor
this.conf.remoteipv6 = this.conf.remoteipv6 === undefined ? this.conf.ipv6 : this.conf.remoteipv6
this.conf.remoteport = this.conf.remoteport === undefined ? this.conf.port : this.conf.remoteport
this.conf.c = this.conf.c === undefined ? constants.CONTRACT.DEFAULT.C : this.conf.c
--
GitLab
From db6f31b29359bc41911741edc8700f1beccb5727 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 04:47:58 +0100
Subject: [PATCH 04/34] add .onion regex
---
app/lib/constants.ts | 3 ++-
app/modules/ws2p/lib/constants.ts | 3 +++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/app/lib/constants.ts b/app/lib/constants.ts
index cac2f7a07..26180e81d 100644
--- a/app/lib/constants.ts
+++ b/app/lib/constants.ts
@@ -109,7 +109,8 @@ module.exports = {
STATUS_INTERVAL: {
UPDATE: 2, // Every X blocks
MAX: 20 // MAX Y blocks
- }
+ },
+ ONION_ENDPOINT_REGEX: new RegExp('(?:https?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
},
PROOF_OF_WORK: {
EVALUATION: 1000,
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index ace1fe96e..59ee7386c 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -8,6 +8,7 @@ export const WS2PConstants = {
CONNEXION_TIMEOUT: 10000,
REQUEST_TIMEOUT: 10000,
+ RECONNEXION_INTERVAL_IN_SEC: 60 * 10, // 10 minutes
BLOCK_PULLING_INTERVAL: 300 * 2, // 10 minutes
DOCPOOL_PULLING_INTERVAL: 3600 * 4, // 4 hours
@@ -36,5 +37,7 @@ export const WS2PConstants = {
+ '(' + CommonConstants.FORMATS.POW_PREFIX + ')'
+ '$'),
+ ONION_ENDPOINT_REGEX: new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*'),
+
HEADS_SPREAD_TIMEOUT: 100 // Wait 100ms before sending a bunch of signed heads
}
\ No newline at end of file
--
GitLab
From 13823417c0795786bd55b8f2215d0cfbcc48d56f Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:03:05 +0100
Subject: [PATCH 05/34] [mod] add custom timeout by proxy
---
app/lib/proxy.ts | 27 +++++++++++++++++++--------
1 file changed, 19 insertions(+), 8 deletions(-)
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index cdcb039dd..c70dcf423 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -3,6 +3,11 @@ const SocksProxyAgent = require('socks-proxy-agent');
const constants = require('./constants');
const WS2PConstants = require('../modules/ws2p/lib/constants');
+const DEFAULT_PROXY_TIMEOUT:number = 30000
+const TOR_PROXY_TIMEOUT:number = 60000
+const HTTP_ENDPOINT_ONION_REGEX = new RegExp('(?:https?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
+const WS_ENDPOINT_ONION_REGEX = new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
+
export interface Proxies {
proxySocks: Proxy|undefined,
proxyTor: Proxy|undefined
@@ -17,20 +22,26 @@ export interface ProxyConf {
export class Proxy {
private agent: any
+ private timeout:number
- constructor(proxy:string, type:string = "socks") {
+ constructor(proxy:string, type:string = "socks", timeout:number = DEFAULT_PROXY_TIMEOUT) {
if (type === "socks") {
this.agent = SocksProxyAgent("socks://"+proxy)
}
else {
this.agent = undefined
}
+ this.timeout = timeout
}
getAgent() {
return this.agent;
}
+ getTimeout() {
+ return this.timeout;
+ }
+
static defaultConf():ProxyConf {
return {
proxySocksAddress: undefined,
@@ -45,7 +56,7 @@ export class Proxy {
if (proxyConf !== undefined) {
return {
proxySocks: (proxyConf.proxySocksAddress !== undefined) ? new Proxy(proxyConf.proxySocksAddress, "socks"):undefined,
- proxyTor: (proxyConf.proxyTorAddress !== undefined) ? new Proxy(proxyConf.proxyTorAddress, "socks"):undefined
+ proxyTor: (proxyConf.proxyTorAddress !== undefined) ? new Proxy(proxyConf.proxyTorAddress, "socks", TOR_PROXY_TIMEOUT):undefined
}
} else {
return undefined
@@ -53,25 +64,25 @@ export class Proxy {
}
static httpProxy(url:string, proxyConf: ProxyConf|undefined) {
- return Proxy.chooseProxy(url, proxyConf, constants.ONION_ENDPOINT_REGEX)
+ return Proxy.chooseProxy(url, proxyConf, HTTP_ENDPOINT_ONION_REGEX)
}
- static wsProxy(address:string, proxyConf: ProxyConf|undefined) {
- return Proxy.chooseProxy(address, proxyConf, WS2PConstants.ONION_ENDPOINT_REGEX)
+ static wsProxy(address:string, proxyConf: ProxyConf|undefined, mySelf:boolean = false) {
+ return Proxy.chooseProxy(address, proxyConf, WS_ENDPOINT_ONION_REGEX, mySelf)
}
- private static chooseProxy(address:string, proxyConf: ProxyConf|undefined, onionRegex:RegExp): Proxy|undefined {
+ private static chooseProxy(address:string, proxyConf: ProxyConf|undefined, onionRegex:RegExp, mySelf:boolean = false): Proxy|undefined {
if (proxyConf !== undefined) {
if (proxyConf.proxies === undefined) {
proxyConf.proxies = Proxy.createProxies(proxyConf)
}
if (proxyConf.proxies !== undefined) {
- if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.getAgent() !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)))
+ if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.getAgent() !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)) && !mySelf )
{
return proxyConf.proxies.proxyTor
}
else if (proxyConf.proxies.proxySocks !== undefined && proxyConf.proxies.proxySocks.getAgent() !== undefined) {
- return proxyConf.proxies.proxyTor
+ return proxyConf.proxies.proxySocks
}
}
}
--
GitLab
From 4e0c3bcb7e4d6ed771ee3a12ce36d0376be46633 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:03:52 +0100
Subject: [PATCH 06/34] move onion regex
---
app/modules/ws2p/lib/constants.ts | 2 --
1 file changed, 2 deletions(-)
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 59ee7386c..ba78a6f1a 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -37,7 +37,5 @@ export const WS2PConstants = {
+ '(' + CommonConstants.FORMATS.POW_PREFIX + ')'
+ '$'),
- ONION_ENDPOINT_REGEX: new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*'),
-
HEADS_SPREAD_TIMEOUT: 100 // Wait 100ms before sending a bunch of signed heads
}
\ No newline at end of file
--
GitLab
From 6b022bce45a6dca72f7c0cb1d30ac0790a6ce397 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:04:29 +0100
Subject: [PATCH 07/34] [enh] sock proxy for ws2p private
---
app/modules/ws2p/lib/WS2PClient.ts | 10 +++++++++-
app/modules/ws2p/lib/WS2PCluster.ts | 4 ++--
app/modules/ws2p/lib/WS2PConnection.ts | 12 ++++++++++--
test/integration/ws2p_connection.ts | 8 ++++----
4 files changed, 25 insertions(+), 9 deletions(-)
diff --git a/app/modules/ws2p/lib/WS2PClient.ts b/app/modules/ws2p/lib/WS2PClient.ts
index 34710012a..0ff0b6178 100644
--- a/app/modules/ws2p/lib/WS2PClient.ts
+++ b/app/modules/ws2p/lib/WS2PClient.ts
@@ -1,3 +1,4 @@
+import { WS2PCluster } from './WS2PCluster';
import {Server} from "../../../../server"
import {WS2PConnection, WS2PPubkeyLocalAuth, WS2PPubkeyRemoteAuth} from "./WS2PConnection"
import {Key} from "../../../lib/common-libs/crypto/keyring"
@@ -5,18 +6,25 @@ import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
import {WS2PConstants} from "./constants"
import {WS2PStreamer} from "./WS2PStreamer"
import {WS2PSingleWriteStream} from "./WS2PSingleWriteStream"
+import { Proxies, ProxyConf, Proxy } from '../../../lib/proxy';
+import { server } from '../../../../test/integration/tools/toolbox';
export class WS2PClient {
private constructor(public connection:WS2PConnection) {}
- static async connectTo(server:Server, fullEndpointAddress:string, messageHandler:WS2PMessageHandler, expectedPub:string, allowKey:(pub:string)=>Promise<boolean> ) {
+ static async connectTo(server:Server, fullEndpointAddress:string, uuid:string, messageHandler:WS2PMessageHandler, expectedPub:string, allowKey:(pub:string)=>Promise<boolean> ) {
const k2 = new Key(server.conf.pair.pub, server.conf.pair.sec)
+ let mySelf = false;
+ if (server.conf.ws2p && server.conf.ws2p.uuid === uuid) {
+ let mySelf = true;
+ }
const c = WS2PConnection.newConnectionToAddress(
fullEndpointAddress,
messageHandler,
new WS2PPubkeyLocalAuth(server.conf.currency , k2, allowKey),
new WS2PPubkeyRemoteAuth(server.conf.currency, k2, allowKey),
+ Proxy.wsProxy(fullEndpointAddress, server.conf.proxyConf, mySelf),
{
connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
requestTimeout: WS2PConstants.REQUEST_TIMEOUT
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index 7f015e75e..caa75b678 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -293,7 +293,7 @@ export class WS2PCluster {
let pub = "--------"
try {
const fullEndpointAddress = WS2PCluster.getFullAddress(host, port, path)
- const ws2pc = await WS2PClient.connectTo(this.server, fullEndpointAddress, messageHandler, expectedPub, (pub:string) => {
+ const ws2pc = await WS2PClient.connectTo(this.server, fullEndpointAddress, ws2pEndpointUUID, messageHandler, expectedPub, (pub:string) => {
const connectedPubkeys = this.getConnectedPubkeys()
return this.acceptPubkey(expectedPub, connectedPubkeys, () => this.clientsCount(), this.maxLevel1Size, (this.server.conf.ws2p && this.server.conf.ws2p.preferedNodes || []), ws2pEndpointUUID)
})
@@ -626,7 +626,7 @@ export class WS2PCluster {
async startCrawling(waitConnection = false) {
// For connectivity
- this.reconnectionInteval = setInterval(() => this.server.push({ ws2p: 'disconnected' }), 1000 * 60 * 10)
+ this.reconnectionInteval = setInterval(() => this.server.push({ ws2p: 'disconnected' }), 1000 * WS2PConstants.RECONNEXION_INTERVAL_IN_SEC)
// For blocks
if (this.syncBlockInterval)
clearInterval(this.syncBlockInterval);
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index e97bde407..7d0b96a02 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -7,6 +7,7 @@ import {MembershipDTO} from "../../../lib/dto/MembershipDTO"
import {TransactionDTO} from "../../../lib/dto/TransactionDTO"
import {PeerDTO} from "../../../lib/dto/PeerDTO"
import {WS2PConstants} from "./constants"
+import { Proxy } from '../../../lib/proxy';
const ws = require('ws')
const nuuid = require('node-uuid');
const logger = require('../../../lib/logger').NewLogger('ws2p')
@@ -267,15 +268,22 @@ export class WS2PConnection {
messageHandler:WS2PMessageHandler,
localAuth:WS2PLocalAuth,
remoteAuth:WS2PRemoteAuth,
+ proxy:Proxy|undefined = undefined,
options:{
- connectionTimeout:number
+ connectionTimeout:number,
requestTimeout:number
} = {
connectionTimeout: REQUEST_TIMEOUT_VALUE,
requestTimeout: REQUEST_TIMEOUT_VALUE
},
expectedPub:string = "") {
- const websocket = new ws(address)
+ if (proxy !== undefined) {
+ options = {
+ connectionTimeout: proxy.getTimeout(),
+ requestTimeout: proxy.getTimeout()
+ }
+ }
+ const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.getAgent() }):new ws(address)
const onWsOpened:Promise<void> = new Promise(res => {
websocket.on('open', () => res())
})
diff --git a/test/integration/ws2p_connection.ts b/test/integration/ws2p_connection.ts
index 894c2ac29..c84d9a42e 100644
--- a/test/integration/ws2p_connection.ts
+++ b/test/integration/ws2p_connection.ts
@@ -95,7 +95,7 @@ describe('WS2P', () => {
it('should refuse the connection if the server does not answer', async () => {
const keypair = new Key('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP')
- const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), {
+ const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), undefined, {
connectionTimeout: 100,
requestTimeout: 100
})
@@ -104,7 +104,7 @@ describe('WS2P', () => {
it('should refuse the connection if the server answers with a wrong signature', async () => {
const keypair = new Key('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP')
- const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), {
+ const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), undefined, {
connectionTimeout: 100,
requestTimeout: 100
})
@@ -113,7 +113,7 @@ describe('WS2P', () => {
it('should refuse the connection if the server refuses our signature', async () => {
const keypair = new Key('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP')
- const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), {
+ const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PPubkeyRemoteAuth(gtest, keypair), undefined, {
connectionTimeout: 100,
requestTimeout: 100
})
@@ -123,7 +123,7 @@ describe('WS2P', () => {
it('should accept the connection if the server answers with a good signature', async () => {
const keypair = new Key('HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP')
- const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PNoRemoteAuth(), {
+ const ws2p = WS2PConnection.newConnectionToAddress('ws://localhost:20903', new WS2PMutedHandler(), new WS2PPubkeyLocalAuth(gtest, keypair), new WS2PNoRemoteAuth(), undefined, {
connectionTimeout: 1000,
requestTimeout: 1000
})
--
GitLab
From ffa0156e8481b08f4149b1dc111be41b45251ea5 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:26:16 +0100
Subject: [PATCH 08/34] [fix] repair test (typage)
---
test/integration/tools/toolbox.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index fcf1e9670..85c317dff 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -170,7 +170,7 @@ export const fakeSyncServer = async (readBlocksMethod:any, readParticularBlockMe
processRequest: () => { /* Does nothing */ }
};
- const fakeServer = await Network.createServersAndListen("Fake Duniter Server", new Server("", true, {}), [{
+ const fakeServer = await Network.createServersAndListen("Fake Duniter Server", new Server("", true, ConfDTO.mock()), [{
ip: host,
port: port
}], NO_HTTP_LOGS, logger, NO_STATIC_PATH, (app:any, httpMethods:any) => {
--
GitLab
From 33ed8c138de076201163f44c08c57561ad0e8ced Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:37:33 +0100
Subject: [PATCH 09/34] add proxyConf
---
app/lib/dto/ConfDTO.ts | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index 361eb49c9..cdedf5037 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -1,4 +1,5 @@
import {CommonConstants} from "../common-libs/constants"
+import { Proxy, ProxyConf } from '../proxy';
const _ = require('underscore');
const constants = require('../constants');
@@ -46,6 +47,7 @@ export interface KeypairConfDTO {
}
export interface NetworkConfDTO {
+ proxyConf: ProxyConf|undefined
nobma: boolean
remoteport: number
remotehost: string|null
@@ -134,6 +136,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public homename: string,
public memory: boolean,
public nobma: boolean,
+ public proxyConf: ProxyConf|undefined,
public ws2p?: {
privateAccess: boolean
publicAccess: boolean
@@ -152,7 +155,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
) {}
static mock() {
- return new ConfDTO("", "", [], [], 0, 0, 0.6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, 0, false, 0, 0, 0, 0, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", 0, "", "", null, false, "", true, true)
+ return new ConfDTO("", "", [], [], 0, 0, 0.6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, 0, false, 0, 0, 0, 0, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", 0, "", "", null, false, "", true, true, undefined)
}
static defaultConf() {
--
GitLab
From 7dd71740114ea4f8c27e03c9974008a7c4d0f039 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:43:40 +0100
Subject: [PATCH 10/34] add proxyConf (just for respect typage)
---
app/modules/bma/lib/network.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/app/modules/bma/lib/network.ts b/app/modules/bma/lib/network.ts
index 33b994afe..ae3459eb9 100644
--- a/app/modules/bma/lib/network.ts
+++ b/app/modules/bma/lib/network.ts
@@ -337,6 +337,7 @@ async function upnpConf (noupnp:boolean, logger:any) {
const publicPort = await getAvailablePort(client)
const privatePort = publicPort
const conf:NetworkConfDTO = {
+ proxyConf: undefined,
nobma: true,
port: privatePort,
ipv4: '127.0.0.1',
--
GitLab
From 927da9f83889f9906a611a79b790dc7a85006a7f Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 18:45:26 +0100
Subject: [PATCH 11/34] add command --rm-proxies
---
app/cli.ts | 1 +
index.ts | 5 +++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/app/cli.ts b/app/cli.ts
index bab8e0d65..187022269 100644
--- a/app/cli.ts
+++ b/app/cli.ts
@@ -53,6 +53,7 @@ export const ExecuteCommand = () => {
.option('--proxy-tor <host:port>', 'Use Tor Socks Proxy')
.option('--tor-always', 'Pass all outgoing requests through the tor network')
.option('--tor-mixed', 'Pass only ".onion" outgoing requests through the tor network. It\'s the default behavior')
+ .option('--rm-proxies', 'Remove all proxies')
.option('--timeout <milliseconds>', 'Timeout to use when contacting peers', parseInt)
.option('--httplogs', 'Enable HTTP logs')
diff --git a/index.ts b/index.ts
index b1d842581..60adec394 100644
--- a/index.ts
+++ b/index.ts
@@ -452,7 +452,8 @@ function commandLineConf(program:any, conf:ConfDTO = ConfDTO.mock()) {
proxySocks: program.proxySocks,
proxyTor: program.proxyTor,
torAlways: program.torAlways,
- torMixed: program.torMixed
+ torMixed: program.torMixed,
+ rmProxies: program.rmProxies
},
logs: {
http: program.httplogs,
@@ -467,7 +468,7 @@ function commandLineConf(program:any, conf:ConfDTO = ConfDTO.mock()) {
};
// Declare proxyConf
- if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.torAlways || cli.proxies.torMixed) {
+ if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.torAlways || cli.proxies.torMixed || cli.proxies.rmProxies) {
conf.proxyConf = Proxy.defaultConf()
}
--
GitLab
From d125e080a10e8cf5d2cf010d70b59eb5f94f9316 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 21:31:26 +0100
Subject: [PATCH 12/34] [fix] persistent proxyConf
---
app/lib/dal/fileDAL.ts | 5 +++++
index.ts | 2 +-
server.ts | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 6281d83fb..3eec9daed 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -15,6 +15,7 @@ import {DBBlock} from "../db/DBBlock"
import {DBMembership} from "./sqliteDAL/MembershipDAL"
import {MerkleDTO} from "../dto/MerkleDTO"
import {CommonConstants} from "../common-libs/constants"
+import { ProxyConf } from '../proxy';
const fs = require('fs')
const path = require('path')
@@ -842,7 +843,11 @@ export class FileDAL {
let conf = ConfDTO.complete(overrideConf || {});
if (!defaultConf) {
const savedConf = await this.confDAL.loadConf();
+ const savedProxyConf = _(savedConf.proxyConf).extend({});
conf = _(savedConf).extend(overrideConf || {});
+ if (overrideConf.proxyConf !== undefined) {} else {
+ conf.proxyConf = _(savedProxyConf).extend({});
+ }
}
if (this.loadConfHook) {
await this.loadConfHook(conf)
diff --git a/index.ts b/index.ts
index 60adec394..ada66317c 100644
--- a/index.ts
+++ b/index.ts
@@ -438,7 +438,7 @@ class Stack {
}
}
-function commandLineConf(program:any, conf:ConfDTO = ConfDTO.mock()) {
+function commandLineConf(program:any, conf:any = {}) {
conf = conf || {};
const cli = {
diff --git a/server.ts b/server.ts
index f91364b26..89772dbf0 100644
--- a/server.ts
+++ b/server.ts
@@ -73,7 +73,7 @@ export class Server extends stream.Duplex implements HookableServer {
TransactionsService:TransactionService
private documentFIFO:GlobalFifoPromise
- constructor(home:string, memoryOnly:boolean, private overrideConf:ConfDTO) {
+ constructor(home:string, memoryOnly:boolean, private overrideConf:any) {
super({ objectMode: true })
this.home = home;
--
GitLab
From 8db926f94186ac63432c0ab8cab7e24c93769966 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sun, 29 Oct 2017 22:56:41 +0100
Subject: [PATCH 13/34] [enh] WS2PTOR endpoint
---
app/lib/common-libs/constants.ts | 2 ++
app/lib/dal/fileDAL.ts | 2 +-
app/lib/dto/PeerDTO.ts | 3 ++-
app/modules/ws2p/index.ts | 30 ++++++++++++++---------
app/modules/ws2p/lib/WS2PCluster.ts | 37 +++++++++++++++++++++++------
app/modules/ws2p/lib/constants.ts | 2 ++
6 files changed, 56 insertions(+), 20 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 14465aade..a49a01138 100644
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -32,6 +32,7 @@ const CONDITIONS = "(&&|\\|\\|| |[()]|(SIG\\(" + PUBKEY + "\\)|(XHX\\([A-F0-9]
const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
const WS2P_REGEXP = /^WS2P ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
+const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+).onion ([0-9]+)(?: (.+))?$/
const IPV4_REGEXP = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IPV6_REGEXP = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
@@ -90,6 +91,7 @@ export const CommonConstants = {
BMA_REGEXP,
WS2P_REGEXP,
+ WS2PTOR_REGEXP,
IPV4_REGEXP,
IPV6_REGEXP,
PUBLIC_KEY: exact(PUBKEY),
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 3eec9daed..9e5865182 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -143,7 +143,7 @@ export class FileDAL {
}
async getWS2Peers() {
- return this.peerDAL.getPeersWithEndpointsLike('WS2P ')
+ return this.peerDAL.getPeersWithEndpointsLike('WS2P ').concat(this.peerDAL.getPeersWithEndpointsLike('WS2PTOR '))
}
async getBlock(number:number) {
diff --git a/app/lib/dto/PeerDTO.ts b/app/lib/dto/PeerDTO.ts
index 738c54063..e2ffcf333 100644
--- a/app/lib/dto/PeerDTO.ts
+++ b/app/lib/dto/PeerDTO.ts
@@ -94,8 +94,9 @@ export class PeerDTO implements Cloneable {
return bma || {};
}
- getWS2P() {
+ getWS2P(tor:boolean = false) {
let api:{ uuid:string, host:string, port:number, path:string }|null = null
+ const endpointRegexp = (tor) ? CommonConstants.WS2PTOR_REGEXP:CommonConstants.WS2P_REGEXP
for (const ep of this.endpoints) {
const matches:any = !api && ep.match(CommonConstants.WS2P_REGEXP)
if (matches) {
diff --git a/app/modules/ws2p/index.ts b/app/modules/ws2p/index.ts
index 53e311b02..6a3dd8fbc 100644
--- a/app/modules/ws2p/index.ts
+++ b/app/modules/ws2p/index.ts
@@ -1,7 +1,8 @@
"use strict";
+import { WS2PConstants } from './lib/constants';
import {ConfDTO, WS2PConfDTO} from "../../lib/dto/ConfDTO"
import {Server} from "../../../server"
-import * as stream from "stream"
+import * as stream from 'stream';
import {WS2PCluster} from "./lib/WS2PCluster"
import {WS2PUpnp} from "./lib/ws2p-upnp"
import {CommonConstants} from "../../lib/common-libs/constants"
@@ -242,22 +243,29 @@ export class WS2PAPI extends stream.Transform {
// If WS2P defined and enabled
if (this.server.conf.ws2p !== undefined && (this.server.conf.ws2p.publicAccess || this.server.conf.ws2p.privateAccess))
{
+ let endpointType = "WS2P"
if (this.server.conf.upnp && this.upnpAPI) {
const config = this.upnpAPI.getCurrentConfig()
- return !config ? '' : ['WS2P', this.server.conf.ws2p.uuid, config.remotehost, config.port].join(' ')
+ if (config) {
+ if (config.remotehost.match(WS2PConstants.HOST_ONION_REGEX)) { endpointType += "TOR"; }
+ return [endpointType, this.server.conf.ws2p.uuid, config.remotehost, config.port].join(' ')
+ } else {
+ return ''
+ }
}
else if (this.server.conf.ws2p.uuid
&& this.server.conf.ws2p.remotehost
&& this.server.conf.ws2p.remoteport) {
- let ep = ['WS2P',
- this.server.conf.ws2p.uuid,
- this.server.conf.ws2p.remotehost,
- this.server.conf.ws2p.remoteport
- ].join(' ')
- if (this.server.conf.ws2p.remotepath) {
- ep += ` ${this.server.conf.ws2p.remotepath}`
- }
- return ep
+ if (this.server.conf.ws2p.remotehost.match(WS2PConstants.HOST_ONION_REGEX)) { endpointType += "TOR"; }
+ let ep = [endpointType,
+ this.server.conf.ws2p.uuid,
+ this.server.conf.ws2p.remotehost,
+ this.server.conf.ws2p.remoteport
+ ].join(' ')
+ if (this.server.conf.ws2p.remotepath) {
+ ep += ` ${this.server.conf.ws2p.remotepath}`
+ }
+ return ep
}
}
return ''
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index caa75b678..5282a1090 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -13,7 +13,7 @@ import {OtherConstants} from "../../../lib/other_constants"
import {Key, verify} from "../../../lib/common-libs/crypto/keyring"
import {WS2PServerMessageHandler} from "./interface/WS2PServerMessageHandler"
import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
-import { CommonConstants } from "../../../lib/common-libs/constants";
+import { CommonConstants } from '../../../lib/common-libs/constants';
import { Package } from "../../../lib/common/package";
import { Constants } from "../../prover/lib/constants";
@@ -330,21 +330,44 @@ export class WS2PCluster {
const prefered = ((this.server.conf.ws2p && this.server.conf.ws2p.preferedNodes) || []).slice() // Copy
// Our key is also a prefered one, so we connect to our siblings
prefered.push(this.server.conf.pair.pub)
+ const imTorPeer = this.server.conf.proxyConf && this.server.conf.proxyConf.proxies && this.server.conf.proxyConf.proxies.proxyTor
peers.sort((a, b) => {
const aIsPrefered = prefered.indexOf(a.pubkey) !== -1
const bIsPrefered = prefered.indexOf(b.pubkey) !== -1
- if ((aIsPrefered && bIsPrefered) || (!aIsPrefered && !bIsPrefered)) {
- return 0
- } else if (aIsPrefered) {
- return -1
+
+ if (imTorPeer) {
+ const aAtWs2pTorEnpoint = a.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
+ const bAtWs2pTorEnpoint = b.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
+
+ if ( (aAtWs2pTorEnpoint && bAtWs2pTorEnpoint) || (!aAtWs2pTorEnpoint && !bAtWs2pTorEnpoint) ) {
+ if ((aIsPrefered && bIsPrefered) || (!aIsPrefered && !bIsPrefered)) {
+ return 0
+ } else if (aIsPrefered) {
+ return -1
+ } else {
+ return 1
+ }
+ } else {
+ if (aAtWs2pTorEnpoint) {
+ return -1
+ } else {
+ return 1
+ }
+ }
} else {
- return 1
+ if ((aIsPrefered && bIsPrefered) || (!aIsPrefered && !bIsPrefered)) {
+ return 0
+ } else if (aIsPrefered) {
+ return -1
+ } else {
+ return 1
+ }
}
})
let i = 0
while (i < peers.length && this.clientsCount() < this.maxLevel1Size) {
const p = peers[i]
- const api = p.getWS2P()
+ const api = p.getWS2P(imTorPeer !== undefined)
if (api) {
try {
await this.connectToRemoteWS(api.host, api.port, api.path, this.messageHandler, p.pubkey, api.uuid)
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index ba78a6f1a..92c69becc 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -37,5 +37,7 @@ export const WS2PConstants = {
+ '(' + CommonConstants.FORMATS.POW_PREFIX + ')'
+ '$'),
+ HOST_ONION_REGEX: new RegExp('(\S*?\.onion)$'),
+
HEADS_SPREAD_TIMEOUT: 100 // Wait 100ms before sending a bunch of signed heads
}
\ No newline at end of file
--
GitLab
From 291f32bcd60b829f17a113e3584ad5c3842af6b9 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Mon, 30 Oct 2017 01:14:22 +0100
Subject: [PATCH 14/34] [fix] get WS2P peers
---
app/lib/dal/fileDAL.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 9e5865182..1298d6a12 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -143,7 +143,7 @@ export class FileDAL {
}
async getWS2Peers() {
- return this.peerDAL.getPeersWithEndpointsLike('WS2P ').concat(this.peerDAL.getPeersWithEndpointsLike('WS2PTOR '))
+ return this.peerDAL.getPeersWithEndpointsLike('WS2P')
}
async getBlock(number:number) {
--
GitLab
From 02883fa265f26156f2047742a33afc35d5961f85 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Mon, 30 Oct 2017 18:08:07 +0100
Subject: [PATCH 15/34] getWS2P apply tor regex
---
app/lib/common-libs/constants.ts | 2 +-
app/lib/dto/PeerDTO.ts | 2 +-
app/lib/proxy.ts | 24 +++++++++++++-----------
app/modules/ws2p/lib/WS2PCluster.ts | 14 ++++++++------
app/modules/ws2p/lib/WS2PConnection.ts | 6 +++---
5 files changed, 26 insertions(+), 22 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index a49a01138..e44639342 100644
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -32,7 +32,7 @@ const CONDITIONS = "(&&|\\|\\|| |[()]|(SIG\\(" + PUBKEY + "\\)|(XHX\\([A-F0-9]
const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
const WS2P_REGEXP = /^WS2P ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
-const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+).onion ([0-9]+)(?: (.+))?$/
+const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+.onion) ([0-9]+)(?: (.+))?$/
const IPV4_REGEXP = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IPV6_REGEXP = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
diff --git a/app/lib/dto/PeerDTO.ts b/app/lib/dto/PeerDTO.ts
index e2ffcf333..d0e661bb8 100644
--- a/app/lib/dto/PeerDTO.ts
+++ b/app/lib/dto/PeerDTO.ts
@@ -98,7 +98,7 @@ export class PeerDTO implements Cloneable {
let api:{ uuid:string, host:string, port:number, path:string }|null = null
const endpointRegexp = (tor) ? CommonConstants.WS2PTOR_REGEXP:CommonConstants.WS2P_REGEXP
for (const ep of this.endpoints) {
- const matches:any = !api && ep.match(CommonConstants.WS2P_REGEXP)
+ const matches:any = !api && ep.match(endpointRegexp)
if (matches) {
api = {
uuid: matches[1],
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index c70dcf423..3627314db 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -21,25 +21,23 @@ export interface ProxyConf {
}
export class Proxy {
- private agent: any
- private timeout:number
+ public agent: any
+ private url:string
- constructor(proxy:string, type:string = "socks", timeout:number = DEFAULT_PROXY_TIMEOUT) {
+ constructor(proxy:string, type:string = "socks", public timeout:number = DEFAULT_PROXY_TIMEOUT) {
if (type === "socks") {
this.agent = SocksProxyAgent("socks://"+proxy)
+ this.url = "socks://"+proxy
}
else {
+ this.url = ""
this.agent = undefined
}
this.timeout = timeout
}
- getAgent() {
- return this.agent;
- }
-
- getTimeout() {
- return this.timeout;
+ getUrl() {
+ return this.url;
}
static defaultConf():ProxyConf {
@@ -51,6 +49,10 @@ export class Proxy {
}
}
+ static canReachTorEndpoint(proxyConf: ProxyConf|undefined):boolean {
+ return (proxyConf !== undefined && (proxyConf.alwaysUseTor === true || (proxyConf.proxies !== undefined && proxyConf.proxies.proxyTor !== undefined) ) )
+ }
+
static createProxies(proxyConf: ProxyConf|undefined) : Proxies|undefined
{
if (proxyConf !== undefined) {
@@ -77,11 +79,11 @@ export class Proxy {
proxyConf.proxies = Proxy.createProxies(proxyConf)
}
if (proxyConf.proxies !== undefined) {
- if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.getAgent() !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)) && !mySelf )
+ if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.agent !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)) && !mySelf )
{
return proxyConf.proxies.proxyTor
}
- else if (proxyConf.proxies.proxySocks !== undefined && proxyConf.proxies.proxySocks.getAgent() !== undefined) {
+ else if (proxyConf.proxies.proxySocks !== undefined && proxyConf.proxies.proxySocks.agent !== undefined) {
return proxyConf.proxies.proxySocks
}
}
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index 5282a1090..d1aa4ece5 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -16,6 +16,7 @@ import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
import { CommonConstants } from '../../../lib/common-libs/constants';
import { Package } from "../../../lib/common/package";
import { Constants } from "../../prover/lib/constants";
+import { ProxyConf, Proxy } from '../../../lib/proxy';
const es = require('event-stream')
const nuuid = require('node-uuid')
@@ -291,6 +292,7 @@ export class WS2PCluster {
async connectToRemoteWS(host: string, port: number, path:string, messageHandler:WS2PMessageHandler, expectedPub:string, ws2pEndpointUUID:string = ""): Promise<WS2PConnection> {
const uuid = nuuid.v4()
let pub = "--------"
+ const api:string = (host.match(WS2PConstants.HOST_ONION_REGEX) !== null) ? 'WS2PTOR':'WS2P'
try {
const fullEndpointAddress = WS2PCluster.getFullAddress(host, port, path)
const ws2pc = await WS2PClient.connectTo(this.server, fullEndpointAddress, ws2pEndpointUUID, messageHandler, expectedPub, (pub:string) => {
@@ -300,7 +302,7 @@ export class WS2PCluster {
this.ws2pClients[uuid] = ws2pc
pub = ws2pc.connection.pubkey
ws2pc.connection.closed.then(() => {
- this.server.logger.info('WS2P: connection [%s `WS2P %s %s`] has been closed', pub.slice(0, 8), host, port)
+ this.server.logger.info(api+': connection [%s `'+api+' %s %s`] has been closed', pub.slice(0, 8), host, port)
this.server.push({
ws2p: 'disconnected',
peer: {
@@ -311,7 +313,7 @@ export class WS2PCluster {
delete this.ws2pClients[uuid]
}
})
- this.server.logger.info('WS2P: connected to peer %s using `WS2P %s %s`!', pub.slice(0, 8), host, port)
+ this.server.logger.info(api+': connected to peer %s using `'+api+' %s %s`!', pub.slice(0, 8), host, port)
this.server.push({
ws2p: 'connected',
to: { host, port, pubkey: pub }
@@ -319,7 +321,7 @@ export class WS2PCluster {
await this.server.dal.setPeerUP(pub)
return ws2pc.connection
} catch (e) {
- this.server.logger.info('WS2P: Could not connect to peer %s using `WS2P %s %s: %s`', pub.slice(0, 8), host, port, (e && e.message || e))
+ this.server.logger.info(api+': Could not connect to peer %s using `'+api+' %s %s: %s`', pub.slice(0, 8), host, port, (e && e.message || e))
throw e
}
}
@@ -330,12 +332,12 @@ export class WS2PCluster {
const prefered = ((this.server.conf.ws2p && this.server.conf.ws2p.preferedNodes) || []).slice() // Copy
// Our key is also a prefered one, so we connect to our siblings
prefered.push(this.server.conf.pair.pub)
- const imTorPeer = this.server.conf.proxyConf && this.server.conf.proxyConf.proxies && this.server.conf.proxyConf.proxies.proxyTor
+ const imCanReachTorEndpoint = Proxy.canReachTorEndpoint(this.server.conf.proxyConf)
peers.sort((a, b) => {
const aIsPrefered = prefered.indexOf(a.pubkey) !== -1
const bIsPrefered = prefered.indexOf(b.pubkey) !== -1
- if (imTorPeer) {
+ if (imCanReachTorEndpoint) {
const aAtWs2pTorEnpoint = a.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
const bAtWs2pTorEnpoint = b.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
@@ -367,7 +369,7 @@ export class WS2PCluster {
let i = 0
while (i < peers.length && this.clientsCount() < this.maxLevel1Size) {
const p = peers[i]
- const api = p.getWS2P(imTorPeer !== undefined)
+ const api = p.getWS2P(imCanReachTorEndpoint !== undefined)
if (api) {
try {
await this.connectToRemoteWS(api.host, api.port, api.path, this.messageHandler, p.pubkey, api.uuid)
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index 7d0b96a02..06de3fc44 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -279,11 +279,11 @@ export class WS2PConnection {
expectedPub:string = "") {
if (proxy !== undefined) {
options = {
- connectionTimeout: proxy.getTimeout(),
- requestTimeout: proxy.getTimeout()
+ connectionTimeout: proxy.timeout,
+ requestTimeout: proxy.timeout
}
}
- const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.getAgent() }):new ws(address)
+ const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.agent }):new ws(address)
const onWsOpened:Promise<void> = new Promise(res => {
websocket.on('open', () => res())
})
--
GitLab
From 3e117452ef60a21af582852878dd8ee37ca70be7 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Mon, 30 Oct 2017 19:23:15 +0100
Subject: [PATCH 16/34] proxy attribute re-encapsulation
---
app/lib/proxy.ts | 24 ++++++++++++++----------
app/modules/ws2p/lib/WS2PConnection.ts | 6 +++---
2 files changed, 17 insertions(+), 13 deletions(-)
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index 3627314db..8c7016890 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -1,8 +1,5 @@
const SocksProxyAgent = require('socks-proxy-agent');
-const constants = require('./constants');
-const WS2PConstants = require('../modules/ws2p/lib/constants');
-
const DEFAULT_PROXY_TIMEOUT:number = 30000
const TOR_PROXY_TIMEOUT:number = 60000
const HTTP_ENDPOINT_ONION_REGEX = new RegExp('(?:https?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
@@ -21,25 +18,32 @@ export interface ProxyConf {
}
export class Proxy {
- public agent: any
+ private agent: any
private url:string
- constructor(proxy:string, type:string = "socks", public timeout:number = DEFAULT_PROXY_TIMEOUT) {
+ constructor(proxy:string, type:string = "socks", private timeout:number = DEFAULT_PROXY_TIMEOUT) {
if (type === "socks") {
- this.agent = SocksProxyAgent("socks://"+proxy)
- this.url = "socks://"+proxy
+ this.agent = SocksProxyAgent("socks://"+proxy)
+ this.url = "socks://"+proxy
}
else {
- this.url = ""
- this.agent = undefined
+ this.agent = undefined
+ this.url = ""
}
- this.timeout = timeout
+ }
+
+ getAgent() {
+ return this.agent;
}
getUrl() {
return this.url;
}
+ getTimeout() {
+ return this.timeout;
+ }
+
static defaultConf():ProxyConf {
return {
proxySocksAddress: undefined,
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index 06de3fc44..7d0b96a02 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -279,11 +279,11 @@ export class WS2PConnection {
expectedPub:string = "") {
if (proxy !== undefined) {
options = {
- connectionTimeout: proxy.timeout,
- requestTimeout: proxy.timeout
+ connectionTimeout: proxy.getTimeout(),
+ requestTimeout: proxy.getTimeout()
}
}
- const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.agent }):new ws(address)
+ const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.getAgent() }):new ws(address)
const onWsOpened:Promise<void> = new Promise(res => {
websocket.on('open', () => res())
})
--
GitLab
From b280d826343788e9f8a9e50ef06f73333cc7ca62 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Mon, 30 Oct 2017 20:43:23 +0100
Subject: [PATCH 17/34] [ifx] getWS2P : return WS2P endpoint if there not
WS2PTOR
---
app/lib/dto/PeerDTO.ts | 13 ++++++++++++-
app/modules/ws2p/index.ts | 2 +-
app/modules/ws2p/lib/WS2PCluster.ts | 2 +-
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/app/lib/dto/PeerDTO.ts b/app/lib/dto/PeerDTO.ts
index d0e661bb8..2bb2e91cd 100644
--- a/app/lib/dto/PeerDTO.ts
+++ b/app/lib/dto/PeerDTO.ts
@@ -98,7 +98,18 @@ export class PeerDTO implements Cloneable {
let api:{ uuid:string, host:string, port:number, path:string }|null = null
const endpointRegexp = (tor) ? CommonConstants.WS2PTOR_REGEXP:CommonConstants.WS2P_REGEXP
for (const ep of this.endpoints) {
- const matches:any = !api && ep.match(endpointRegexp)
+ if (tor) {
+ const matches:any = ep.match(CommonConstants.WS2PTOR_REGEXP)
+ if (matches) {
+ return {
+ uuid: matches[1],
+ host: matches[2] || '',
+ port: parseInt(matches[3]) || 0,
+ path: matches[4]
+ }
+ }
+ }
+ const matches:any = !api && ep.match(CommonConstants.WS2P_REGEXP)
if (matches) {
api = {
uuid: matches[1],
diff --git a/app/modules/ws2p/index.ts b/app/modules/ws2p/index.ts
index 6a3dd8fbc..a225e63ec 100644
--- a/app/modules/ws2p/index.ts
+++ b/app/modules/ws2p/index.ts
@@ -141,7 +141,7 @@ export const WS2PDependency = {
const peers = await server.dal.getWS2Peers()
for (const p of peers) {
for (const ep of p.endpoints) {
- if (ep.match(/^WS2P /)) {
+ if (ep.match(/^WS2P/)) {
console.log(p.pubkey, ep)
}
}
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index d1aa4ece5..93e8119d8 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -369,7 +369,7 @@ export class WS2PCluster {
let i = 0
while (i < peers.length && this.clientsCount() < this.maxLevel1Size) {
const p = peers[i]
- const api = p.getWS2P(imCanReachTorEndpoint !== undefined)
+ const api = p.getWS2P(imCanReachTorEndpoint)
if (api) {
try {
await this.connectToRemoteWS(api.host, api.port, api.path, this.messageHandler, p.pubkey, api.uuid)
--
GitLab
From 8198ed468bcbb3faf01b4d4d54568b7975ea9aa9 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:01:20 +0100
Subject: [PATCH 18/34] rename ProxyConf -> ProxiesConf
---
app/lib/dal/fileDAL.ts | 2 +-
app/lib/dto/ConfDTO.ts | 6 +++---
app/modules/ws2p/lib/WS2PClient.ts | 8 ++------
app/modules/ws2p/lib/WS2PCluster.ts | 4 ++--
4 files changed, 8 insertions(+), 12 deletions(-)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 1298d6a12..267d0f184 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -15,7 +15,7 @@ import {DBBlock} from "../db/DBBlock"
import {DBMembership} from "./sqliteDAL/MembershipDAL"
import {MerkleDTO} from "../dto/MerkleDTO"
import {CommonConstants} from "../common-libs/constants"
-import { ProxyConf } from '../proxy';
+import { ProxiesConf } from '../proxy';
const fs = require('fs')
const path = require('path')
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index cdedf5037..fdd2038b7 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -1,5 +1,5 @@
import {CommonConstants} from "../common-libs/constants"
-import { Proxy, ProxyConf } from '../proxy';
+import { ProxiesConf } from '../proxy';
const _ = require('underscore');
const constants = require('../constants');
@@ -47,7 +47,7 @@ export interface KeypairConfDTO {
}
export interface NetworkConfDTO {
- proxyConf: ProxyConf|undefined
+ proxyConf: ProxiesConf|undefined
nobma: boolean
remoteport: number
remotehost: string|null
@@ -136,7 +136,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public homename: string,
public memory: boolean,
public nobma: boolean,
- public proxyConf: ProxyConf|undefined,
+ public proxyConf: ProxiesConf|undefined,
public ws2p?: {
privateAccess: boolean
publicAccess: boolean
diff --git a/app/modules/ws2p/lib/WS2PClient.ts b/app/modules/ws2p/lib/WS2PClient.ts
index 0ff0b6178..80a4fd1bd 100644
--- a/app/modules/ws2p/lib/WS2PClient.ts
+++ b/app/modules/ws2p/lib/WS2PClient.ts
@@ -6,7 +6,7 @@ import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
import {WS2PConstants} from "./constants"
import {WS2PStreamer} from "./WS2PStreamer"
import {WS2PSingleWriteStream} from "./WS2PSingleWriteStream"
-import { Proxies, ProxyConf, Proxy } from '../../../lib/proxy';
+import { ProxiesConf } from '../../../lib/proxy';
import { server } from '../../../../test/integration/tools/toolbox';
export class WS2PClient {
@@ -15,16 +15,12 @@ export class WS2PClient {
static async connectTo(server:Server, fullEndpointAddress:string, uuid:string, messageHandler:WS2PMessageHandler, expectedPub:string, allowKey:(pub:string)=>Promise<boolean> ) {
const k2 = new Key(server.conf.pair.pub, server.conf.pair.sec)
- let mySelf = false;
- if (server.conf.ws2p && server.conf.ws2p.uuid === uuid) {
- let mySelf = true;
- }
const c = WS2PConnection.newConnectionToAddress(
fullEndpointAddress,
messageHandler,
new WS2PPubkeyLocalAuth(server.conf.currency , k2, allowKey),
new WS2PPubkeyRemoteAuth(server.conf.currency, k2, allowKey),
- Proxy.wsProxy(fullEndpointAddress, server.conf.proxyConf, mySelf),
+ ProxiesConf.wsProxy(fullEndpointAddress, server.conf.proxyConf),
{
connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
requestTimeout: WS2PConstants.REQUEST_TIMEOUT
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index 93e8119d8..5e785d9ef 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -16,7 +16,7 @@ import {WS2PMessageHandler} from "./impl/WS2PMessageHandler"
import { CommonConstants } from '../../../lib/common-libs/constants';
import { Package } from "../../../lib/common/package";
import { Constants } from "../../prover/lib/constants";
-import { ProxyConf, Proxy } from '../../../lib/proxy';
+import { ProxiesConf } from '../../../lib/proxy';
const es = require('event-stream')
const nuuid = require('node-uuid')
@@ -332,7 +332,7 @@ export class WS2PCluster {
const prefered = ((this.server.conf.ws2p && this.server.conf.ws2p.preferedNodes) || []).slice() // Copy
// Our key is also a prefered one, so we connect to our siblings
prefered.push(this.server.conf.pair.pub)
- const imCanReachTorEndpoint = Proxy.canReachTorEndpoint(this.server.conf.proxyConf)
+ const imCanReachTorEndpoint = ProxiesConf.canReachTorEndpoint(this.server.conf.proxyConf)
peers.sort((a, b) => {
const aIsPrefered = prefered.indexOf(a.pubkey) !== -1
const bIsPrefered = prefered.indexOf(b.pubkey) !== -1
--
GitLab
From 89a716bfac62b6a7a521ceca25f00cbdc39eea88 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:02:43 +0100
Subject: [PATCH 19/34] rewrite proxiesConf type
---
app/lib/proxy.ts | 104 +++++++++++------------------------------------
index.ts | 4 +-
server.ts | 4 +-
3 files changed, 28 insertions(+), 84 deletions(-)
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index 8c7016890..051a57d3a 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -1,97 +1,41 @@
const SocksProxyAgent = require('socks-proxy-agent');
-const DEFAULT_PROXY_TIMEOUT:number = 30000
-const TOR_PROXY_TIMEOUT:number = 60000
-const HTTP_ENDPOINT_ONION_REGEX = new RegExp('(?:https?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
-const WS_ENDPOINT_ONION_REGEX = new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*')
+const HOST_ONION_REGEX = new RegExp('(\S*?\.onion)$');
+const WS_ENDPOINT_ONION_REGEX = new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*');
-export interface Proxies {
- proxySocks: Proxy|undefined,
- proxyTor: Proxy|undefined
-}
+export class ProxiesConf {
+ public proxySocksAddress: string|undefined
+ public proxyTorAddress: string|undefined
+ public alwaysUseTor: boolean|undefined
-export interface ProxyConf {
- proxySocksAddress: string|undefined,
- proxyTorAddress: string|undefined,
- alwaysUseTor: boolean|undefined,
- proxies: Proxies|undefined
-}
-
-export class Proxy {
- private agent: any
- private url:string
-
- constructor(proxy:string, type:string = "socks", private timeout:number = DEFAULT_PROXY_TIMEOUT) {
- if (type === "socks") {
- this.agent = SocksProxyAgent("socks://"+proxy)
- this.url = "socks://"+proxy
- }
- else {
- this.agent = undefined
- this.url = ""
- }
+ constructor () {
+ this.proxySocksAddress = undefined
+ this.proxyTorAddress = undefined
+ this.alwaysUseTor = undefined
}
- getAgent() {
- return this.agent;
+ static canReachTorEndpoint(proxyConf: ProxiesConf|undefined):boolean {
+ return (proxyConf !== undefined && (proxyConf.alwaysUseTor === true || (proxyConf.proxyTorAddress !== undefined) ) )
}
- getUrl() {
- return this.url;
- }
-
- getTimeout() {
- return this.timeout;
- }
-
- static defaultConf():ProxyConf {
- return {
- proxySocksAddress: undefined,
- proxyTorAddress: undefined,
- alwaysUseTor: undefined,
- proxies: undefined
- }
+ static httpProxy(url:string, proxyConf: ProxiesConf|undefined):string|undefined {
+ return ProxiesConf.chooseProxyAgent(url, proxyConf, HOST_ONION_REGEX)
}
- static canReachTorEndpoint(proxyConf: ProxyConf|undefined):boolean {
- return (proxyConf !== undefined && (proxyConf.alwaysUseTor === true || (proxyConf.proxies !== undefined && proxyConf.proxies.proxyTor !== undefined) ) )
+ static wsProxy(address:string, proxyConf: ProxiesConf|undefined):string|undefined {
+ return ProxiesConf.chooseProxyAgent(address, proxyConf, WS_ENDPOINT_ONION_REGEX)
}
- static createProxies(proxyConf: ProxyConf|undefined) : Proxies|undefined
- {
+ private static chooseProxyAgent(address:string, proxyConf: ProxiesConf|undefined, onionRegex:RegExp):string|undefined {
if (proxyConf !== undefined) {
- return {
- proxySocks: (proxyConf.proxySocksAddress !== undefined) ? new Proxy(proxyConf.proxySocksAddress, "socks"):undefined,
- proxyTor: (proxyConf.proxyTorAddress !== undefined) ? new Proxy(proxyConf.proxyTorAddress, "socks", TOR_PROXY_TIMEOUT):undefined
+ if ( proxyConf.proxyTorAddress !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)))
+ {
+ return proxyConf.proxyTorAddress
+ }
+ else if (proxyConf.proxySocksAddress !== undefined) {
+ return proxyConf.proxySocksAddress
}
- } else {
- return undefined
- }
- }
-
- static httpProxy(url:string, proxyConf: ProxyConf|undefined) {
- return Proxy.chooseProxy(url, proxyConf, HTTP_ENDPOINT_ONION_REGEX)
- }
-
- static wsProxy(address:string, proxyConf: ProxyConf|undefined, mySelf:boolean = false) {
- return Proxy.chooseProxy(address, proxyConf, WS_ENDPOINT_ONION_REGEX, mySelf)
- }
-
- private static chooseProxy(address:string, proxyConf: ProxyConf|undefined, onionRegex:RegExp, mySelf:boolean = false): Proxy|undefined {
- if (proxyConf !== undefined) {
- if (proxyConf.proxies === undefined) {
- proxyConf.proxies = Proxy.createProxies(proxyConf)
- }
- if (proxyConf.proxies !== undefined) {
- if ( proxyConf.proxies.proxyTor !== undefined && proxyConf.proxies.proxyTor.agent !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)) && !mySelf )
- {
- return proxyConf.proxies.proxyTor
- }
- else if (proxyConf.proxies.proxySocks !== undefined && proxyConf.proxies.proxySocks.agent !== undefined) {
- return proxyConf.proxies.proxySocks
- }
- }
}
return undefined
}
-}
+}
\ No newline at end of file
diff --git a/index.ts b/index.ts
index ada66317c..70b7bd28a 100644
--- a/index.ts
+++ b/index.ts
@@ -8,7 +8,7 @@ import {CrawlerDependency} from "./app/modules/crawler/index"
import {BmaDependency} from "./app/modules/bma/index"
import {WS2PDependency} from "./app/modules/ws2p/index"
import {Constants} from "./app/modules/prover/lib/constants"
-import { Proxy } from './app/lib/proxy';
+import { ProxiesConf } from './app/lib/proxy';
const path = require('path');
const _ = require('underscore');
@@ -469,7 +469,7 @@ function commandLineConf(program:any, conf:any = {}) {
// Declare proxyConf
if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.torAlways || cli.proxies.torMixed || cli.proxies.rmProxies) {
- conf.proxyConf = Proxy.defaultConf()
+ conf.proxyConf = new ProxiesConf()
}
// Update conf
diff --git a/server.ts b/server.ts
index 89772dbf0..9eca4b855 100644
--- a/server.ts
+++ b/server.ts
@@ -24,7 +24,7 @@ import {PeerDTO} from "./app/lib/dto/PeerDTO"
import {OtherConstants} from "./app/lib/other_constants"
import {WS2PCluster} from "./app/modules/ws2p/lib/WS2PCluster"
import {DBBlock} from "./app/lib/db/DBBlock"
-import { Proxy } from './app/lib/proxy';
+import { ProxiesConf } from './app/lib/proxy';
export interface HookableServer {
generatorGetJoinData: (...args:any[]) => Promise<any>
@@ -149,7 +149,7 @@ export class Server extends stream.Duplex implements HookableServer {
logger.debug('Loading conf...');
this.conf = await this.dal.loadConf(this.overrideConf, useDefaultConf)
// Default values
- this.conf.proxyConf = this.conf.proxyConf === undefined ? Proxy.defaultConf() : this.conf.proxyConf
+ this.conf.proxyConf = this.conf.proxyConf === undefined ? new ProxiesConf() : this.conf.proxyConf
this.conf.proxyConf.alwaysUseTor = this.conf.proxyConf.alwaysUseTor === undefined ? false : this.conf.proxyConf.alwaysUseTor
this.conf.remoteipv6 = this.conf.remoteipv6 === undefined ? this.conf.ipv6 : this.conf.remoteipv6
this.conf.remoteport = this.conf.remoteport === undefined ? this.conf.port : this.conf.remoteport
--
GitLab
From 87721f1e963a05276c6a0558e76c95aa4e0e142f Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:03:44 +0100
Subject: [PATCH 20/34] add constant PROXY_TIMEOUT
---
app/modules/ws2p/lib/constants.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 92c69becc..6a5e93f24 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -8,6 +8,7 @@ export const WS2PConstants = {
CONNEXION_TIMEOUT: 10000,
REQUEST_TIMEOUT: 10000,
+ PROXY_TIMEOUT: 30000,
RECONNEXION_INTERVAL_IN_SEC: 60 * 10, // 10 minutes
BLOCK_PULLING_INTERVAL: 300 * 2, // 10 minutes
--
GitLab
From d0acedd5a1264c95ed6d32e53bfc91f0812df94e Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:05:36 +0100
Subject: [PATCH 21/34] change proxy agent application
---
app/modules/ws2p/lib/WS2PConnection.ts | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index 7d0b96a02..9a08ac742 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -7,8 +7,9 @@ import {MembershipDTO} from "../../../lib/dto/MembershipDTO"
import {TransactionDTO} from "../../../lib/dto/TransactionDTO"
import {PeerDTO} from "../../../lib/dto/PeerDTO"
import {WS2PConstants} from "./constants"
-import { Proxy } from '../../../lib/proxy';
+import { ProxiesConf } from '../../../lib/proxy';
const ws = require('ws')
+const SocksProxyAgent = require('socks-proxy-agent');
const nuuid = require('node-uuid');
const logger = require('../../../lib/logger').NewLogger('ws2p')
@@ -268,7 +269,7 @@ export class WS2PConnection {
messageHandler:WS2PMessageHandler,
localAuth:WS2PLocalAuth,
remoteAuth:WS2PRemoteAuth,
- proxy:Proxy|undefined = undefined,
+ proxySocksAddress:string|undefined = undefined,
options:{
connectionTimeout:number,
requestTimeout:number
@@ -277,13 +278,13 @@ export class WS2PConnection {
requestTimeout: REQUEST_TIMEOUT_VALUE
},
expectedPub:string = "") {
- if (proxy !== undefined) {
+ if (proxySocksAddress !== undefined) {
options = {
- connectionTimeout: proxy.getTimeout(),
- requestTimeout: proxy.getTimeout()
+ connectionTimeout: WS2PConstants.PROXY_TIMEOUT,
+ requestTimeout: WS2PConstants.PROXY_TIMEOUT
}
}
- const websocket = (proxy !== undefined) ? new ws(address, { agent: proxy.getAgent() }):new ws(address)
+ const websocket = (proxySocksAddress !== undefined) ? new ws(address, { agent: SocksProxyAgent("socks://"+proxySocksAddress) }):new ws(address)
const onWsOpened:Promise<void> = new Promise(res => {
websocket.on('open', () => res())
})
--
GitLab
From 1174af18c7ee602d9010619cde287720abbb028f Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:07:34 +0100
Subject: [PATCH 22/34] ignore proxies test compiled files
---
.gitignore | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitignore b/.gitignore
index 3b71ae8db..8d7e9c280 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,5 +54,7 @@ test/fast/modules/crawler/block_pulling.js*
test/fast/modules/crawler/block_pulling.d.ts
test/fast/fork*.js*
test/fast/fork*.d.ts
+test/fast/proxies*.js*
+test/fast/proxies*.d.ts
test/fast/modules/ws2p/*.js*
test/fast/modules/ws2p/*.d.ts
--
GitLab
From e1f8f6f7fc42a60bd21413a9ff59e25f8275e59d Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Tue, 31 Oct 2017 03:09:02 +0100
Subject: [PATCH 23/34] add proxies conf unit test
---
test/fast/proxies.ts | 51 ++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
create mode 100644 test/fast/proxies.ts
diff --git a/test/fast/proxies.ts b/test/fast/proxies.ts
new file mode 100644
index 000000000..c47468092
--- /dev/null
+++ b/test/fast/proxies.ts
@@ -0,0 +1,51 @@
+import * as assert from 'assert'
+import { ProxiesConf } from '../../app/lib/proxy';
+
+describe("Proxies Conf", function() {
+
+ // First conf : do not use any sock proxy
+ let proxiesConf1 = new ProxiesConf()
+
+ // Second conf : use tor only to reach ".onion" endpoints
+ let proxiesConf2 = new ProxiesConf()
+ proxiesConf2.proxyTorAddress = "127.0.0.1:9050"
+
+ // Third conf : always use tor
+ let proxiesConf3 = new ProxiesConf()
+ proxiesConf3.proxyTorAddress = "127.0.0.1:9050"
+ proxiesConf3.alwaysUseTor = true
+
+ // Fourth cont : use classical socks proxy
+ let proxiesConf4 = new ProxiesConf()
+ proxiesConf4.proxySocksAddress = "127.0.0.1:8888"
+
+ // Fifth : use classical socks proxy + use tor proxy only to reach ".onion" endpoints
+ let proxiesConf5 = new ProxiesConf()
+ proxiesConf5.proxySocksAddress = "127.0.0.1:8888"
+ proxiesConf5.proxyTorAddress = "127.0.0.1:9050"
+
+ it('should do not use any sock proxy', () => {
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf1) === undefined, true)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf1) === undefined, true)
+ })
+
+ it('should use tor proxy only to reach ".onion" endpoints', () => {
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf2) === proxiesConf2.proxyTorAddress, true)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf2) === undefined, true)
+ })
+
+ it('should always use tor proxy', () => {
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf3) === proxiesConf3.proxyTorAddress, true)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf3) === proxiesConf3.proxyTorAddress, true)
+ })
+
+ it('should always use classical socks proxy', () => {
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf4) === proxiesConf4.proxySocksAddress, true)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf4) === proxiesConf4.proxySocksAddress, true)
+ })
+
+ it('should use or tor proxy for ".onion" endpoints and classical socks proxy for everyone else', () => {
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf5) === proxiesConf5.proxyTorAddress, true)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf5) === proxiesConf5.proxySocksAddress, true)
+ })
+});
--
GitLab
From f231daf660e08430b029cbd93744332a0f0e7de0 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 03:34:12 +0100
Subject: [PATCH 24/34] [enh] improving tor config and timeout management
---
app/cli.ts | 8 +--
app/lib/dal/fileDAL.ts | 2 +-
app/lib/dto/ConfDTO.ts | 4 +-
app/lib/dto/PeerDTO.ts | 22 ++++----
app/lib/proxy.ts | 48 ++++++++++-------
app/modules/bma/lib/network.ts | 2 +-
app/modules/ws2p/lib/WS2PClient.ts | 2 +-
app/modules/ws2p/lib/WS2PCluster.ts | 9 ++--
app/modules/ws2p/lib/WS2PConnection.ts | 6 +--
app/modules/ws2p/lib/WS2PServer.ts | 15 ++++--
app/modules/ws2p/lib/constants.ts | 10 ++--
index.ts | 29 +++++-----
server.ts | 3 +-
test/fast/proxies.ts | 75 +++++++++++++++++++++-----
14 files changed, 155 insertions(+), 80 deletions(-)
diff --git a/app/cli.ts b/app/cli.ts
index 187022269..26accf24f 100644
--- a/app/cli.ts
+++ b/app/cli.ts
@@ -49,10 +49,10 @@ export const ExecuteCommand = () => {
.option('--nostdout', 'Disable stdout printing for `export-bc` command')
.option('--noshuffle', 'Disable peers shuffling for `sync` command')
- .option('--proxy-socks <host:port>', 'Use Socks Proxy')
- .option('--proxy-tor <host:port>', 'Use Tor Socks Proxy')
- .option('--tor-always', 'Pass all outgoing requests through the tor network')
- .option('--tor-mixed', 'Pass only ".onion" outgoing requests through the tor network. It\'s the default behavior')
+ .option('--socks-proxy <host:port>', 'Use Socks Proxy')
+ .option('--tor-proxy <host:port>', 'Use Tor Socks Proxy')
+ .option('--reaching-clear-ep <clear|tor|none>', 'method for reaching an clear endpoint')
+ .option('--force-tor', 'force duniter to contact endpoint tor (if you redirect the traffic to tor yourself)')
.option('--rm-proxies', 'Remove all proxies')
.option('--timeout <milliseconds>', 'Timeout to use when contacting peers', parseInt)
diff --git a/app/lib/dal/fileDAL.ts b/app/lib/dal/fileDAL.ts
index 267d0f184..600226123 100644
--- a/app/lib/dal/fileDAL.ts
+++ b/app/lib/dal/fileDAL.ts
@@ -845,7 +845,7 @@ export class FileDAL {
const savedConf = await this.confDAL.loadConf();
const savedProxyConf = _(savedConf.proxyConf).extend({});
conf = _(savedConf).extend(overrideConf || {});
- if (overrideConf.proxyConf !== undefined) {} else {
+ if (overrideConf.proxiesConf !== undefined) {} else {
conf.proxyConf = _(savedProxyConf).extend({});
}
}
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index fdd2038b7..1227913e3 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -47,7 +47,7 @@ export interface KeypairConfDTO {
}
export interface NetworkConfDTO {
- proxyConf: ProxiesConf|undefined
+ proxiesConf: ProxiesConf|undefined
nobma: boolean
remoteport: number
remotehost: string|null
@@ -136,7 +136,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public homename: string,
public memory: boolean,
public nobma: boolean,
- public proxyConf: ProxiesConf|undefined,
+ public proxiesConf: ProxiesConf|undefined,
public ws2p?: {
privateAccess: boolean
publicAccess: boolean
diff --git a/app/lib/dto/PeerDTO.ts b/app/lib/dto/PeerDTO.ts
index 2bb2e91cd..dc5754b68 100644
--- a/app/lib/dto/PeerDTO.ts
+++ b/app/lib/dto/PeerDTO.ts
@@ -94,11 +94,11 @@ export class PeerDTO implements Cloneable {
return bma || {};
}
- getWS2P(tor:boolean = false) {
+ getWS2P(canReachTorEp:boolean, canReachClearEp:boolean) {
let api:{ uuid:string, host:string, port:number, path:string }|null = null
- const endpointRegexp = (tor) ? CommonConstants.WS2PTOR_REGEXP:CommonConstants.WS2P_REGEXP
+ const endpointRegexp = (canReachTorEp) ? CommonConstants.WS2PTOR_REGEXP:CommonConstants.WS2P_REGEXP
for (const ep of this.endpoints) {
- if (tor) {
+ if (canReachTorEp) {
const matches:any = ep.match(CommonConstants.WS2PTOR_REGEXP)
if (matches) {
return {
@@ -109,13 +109,15 @@ export class PeerDTO implements Cloneable {
}
}
}
- const matches:any = !api && ep.match(CommonConstants.WS2P_REGEXP)
- if (matches) {
- api = {
- uuid: matches[1],
- host: matches[2] || '',
- port: parseInt(matches[3]) || 0,
- path: matches[4]
+ if (canReachClearEp) {
+ const matches:any = !api && ep.match(CommonConstants.WS2P_REGEXP)
+ if (matches) {
+ api = {
+ uuid: matches[1],
+ host: matches[2] || '',
+ port: parseInt(matches[3]) || 0,
+ path: matches[4]
+ }
}
}
}
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index 051a57d3a..43b000a22 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -1,39 +1,51 @@
const SocksProxyAgent = require('socks-proxy-agent');
-const HOST_ONION_REGEX = new RegExp('(\S*?\.onion)$');
-const WS_ENDPOINT_ONION_REGEX = new RegExp('(?:wss?:\/\/)?(?:www)?(\S*?\.onion)(\/[-\w]*)*');
+const HOST_ONION_REGEX = new RegExp('(?:www\.)?([0-9a-z]{16}?\.onion)$');
+const WS_FULL_ADDRESS_ONION_REGEX = new RegExp('^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*');
export class ProxiesConf {
public proxySocksAddress: string|undefined
public proxyTorAddress: string|undefined
- public alwaysUseTor: boolean|undefined
+ public reachingClearEp: string
+ public forceTor: boolean
constructor () {
this.proxySocksAddress = undefined
this.proxyTorAddress = undefined
- this.alwaysUseTor = undefined
+ this.reachingClearEp = 'clear'
+ this.forceTor = false
}
- static canReachTorEndpoint(proxyConf: ProxiesConf|undefined):boolean {
- return (proxyConf !== undefined && (proxyConf.alwaysUseTor === true || (proxyConf.proxyTorAddress !== undefined) ) )
+ static canReachClearEndpoint(proxiesConf: ProxiesConf|undefined):boolean {
+ return (proxiesConf === undefined || proxiesConf.reachingClearEp !== 'none')
}
- static httpProxy(url:string, proxyConf: ProxiesConf|undefined):string|undefined {
- return ProxiesConf.chooseProxyAgent(url, proxyConf, HOST_ONION_REGEX)
+ static canReachTorEndpoint(proxiesConf: ProxiesConf|undefined):boolean {
+ return (proxiesConf !== undefined && (proxiesConf.forceTor || proxiesConf.proxyTorAddress !== undefined) )
}
- static wsProxy(address:string, proxyConf: ProxiesConf|undefined):string|undefined {
- return ProxiesConf.chooseProxyAgent(address, proxyConf, WS_ENDPOINT_ONION_REGEX)
+ static httpProxy(url:string, proxiesConf: ProxiesConf|undefined):string|undefined {
+ return ProxiesConf.chooseProxyAgent(url, proxiesConf, HOST_ONION_REGEX)
}
- private static chooseProxyAgent(address:string, proxyConf: ProxiesConf|undefined, onionRegex:RegExp):string|undefined {
- if (proxyConf !== undefined) {
- if ( proxyConf.proxyTorAddress !== undefined && (proxyConf.alwaysUseTor || address.match(onionRegex)))
- {
- return proxyConf.proxyTorAddress
- }
- else if (proxyConf.proxySocksAddress !== undefined) {
- return proxyConf.proxySocksAddress
+ static wsProxy(address:string, proxiesConf: ProxiesConf|undefined):string|undefined {
+ return ProxiesConf.chooseProxyAgent(address, proxiesConf, WS_FULL_ADDRESS_ONION_REGEX)
+ }
+
+ private static chooseProxyAgent(address:string, proxiesConf: ProxiesConf|undefined, onionRegex:RegExp):string|undefined {
+ if (proxiesConf !== undefined) {
+ if (address.match(onionRegex)) {
+ if (ProxiesConf.canReachTorEndpoint(proxiesConf)) {
+ return proxiesConf.proxyTorAddress
+ }
+ } else {
+ if (ProxiesConf.canReachClearEndpoint(proxiesConf)) {
+ if (proxiesConf.reachingClearEp == 'tor') {
+ return proxiesConf.proxyTorAddress
+ } else {
+ return proxiesConf.proxySocksAddress
+ }
+ }
}
}
return undefined
diff --git a/app/modules/bma/lib/network.ts b/app/modules/bma/lib/network.ts
index ae3459eb9..250a622bf 100644
--- a/app/modules/bma/lib/network.ts
+++ b/app/modules/bma/lib/network.ts
@@ -337,7 +337,7 @@ async function upnpConf (noupnp:boolean, logger:any) {
const publicPort = await getAvailablePort(client)
const privatePort = publicPort
const conf:NetworkConfDTO = {
- proxyConf: undefined,
+ proxiesConf: undefined,
nobma: true,
port: privatePort,
ipv4: '127.0.0.1',
diff --git a/app/modules/ws2p/lib/WS2PClient.ts b/app/modules/ws2p/lib/WS2PClient.ts
index 80a4fd1bd..569609f6e 100644
--- a/app/modules/ws2p/lib/WS2PClient.ts
+++ b/app/modules/ws2p/lib/WS2PClient.ts
@@ -20,7 +20,7 @@ export class WS2PClient {
messageHandler,
new WS2PPubkeyLocalAuth(server.conf.currency , k2, allowKey),
new WS2PPubkeyRemoteAuth(server.conf.currency, k2, allowKey),
- ProxiesConf.wsProxy(fullEndpointAddress, server.conf.proxyConf),
+ ProxiesConf.wsProxy(fullEndpointAddress, server.conf.proxiesConf),
{
connectionTimeout: WS2PConstants.REQUEST_TIMEOUT,
requestTimeout: WS2PConstants.REQUEST_TIMEOUT
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index 5cc022579..875c552dd 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -332,12 +332,12 @@ export class WS2PCluster {
const prefered = ((this.server.conf.ws2p && this.server.conf.ws2p.preferedNodes) || []).slice() // Copy
// Our key is also a prefered one, so we connect to our siblings
prefered.push(this.server.conf.pair.pub)
- const imCanReachTorEndpoint = ProxiesConf.canReachTorEndpoint(this.server.conf.proxyConf)
+ const canReachTorEndpoint = ProxiesConf.canReachTorEndpoint(this.server.conf.proxiesConf)
peers.sort((a, b) => {
const aIsPrefered = prefered.indexOf(a.pubkey) !== -1
const bIsPrefered = prefered.indexOf(b.pubkey) !== -1
- if (imCanReachTorEndpoint) {
+ if (canReachTorEndpoint) {
const aAtWs2pTorEnpoint = a.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
const bAtWs2pTorEnpoint = b.endpoints.filter(function (element) { return element.match(CommonConstants.WS2PTOR_REGEXP); }).length > 0
@@ -367,9 +367,10 @@ export class WS2PCluster {
}
})
let i = 0
+ const canReachClearEndpoint = ProxiesConf.canReachClearEndpoint(this.server.conf.proxiesConf)
while (i < peers.length && this.clientsCount() < this.maxLevel1Size) {
const p = peers[i]
- const api = p.getWS2P(imCanReachTorEndpoint)
+ const api = p.getWS2P(canReachTorEndpoint, canReachClearEndpoint)
if (api) {
try {
// We do not connect to local host
@@ -396,7 +397,7 @@ export class WS2PCluster {
// New peer
if (data.endpoints) {
const peer = PeerDTO.fromJSONObject(data)
- const ws2pEnpoint = peer.getWS2P()
+ const ws2pEnpoint = peer.getWS2P(ProxiesConf.canReachTorEndpoint(this.server.conf.proxiesConf), ProxiesConf.canReachClearEndpoint(this.server.conf.proxiesConf))
if (ws2pEnpoint) {
// Check if already connected to the pubkey (in any way: server or client)
const connectedPubkeys = this.getConnectedPubkeys()
diff --git a/app/modules/ws2p/lib/WS2PConnection.ts b/app/modules/ws2p/lib/WS2PConnection.ts
index 9a08ac742..0595e5626 100644
--- a/app/modules/ws2p/lib/WS2PConnection.ts
+++ b/app/modules/ws2p/lib/WS2PConnection.ts
@@ -278,10 +278,10 @@ export class WS2PConnection {
requestTimeout: REQUEST_TIMEOUT_VALUE
},
expectedPub:string = "") {
- if (proxySocksAddress !== undefined) {
+ if (address.match(WS2PConstants.FULL_ADDRESS_ONION_REGEX)) {
options = {
- connectionTimeout: WS2PConstants.PROXY_TIMEOUT,
- requestTimeout: WS2PConstants.PROXY_TIMEOUT
+ connectionTimeout: WS2PConstants.CONNEXION_TOR_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TOR_TIMEOUT
}
}
const websocket = (proxySocksAddress !== undefined) ? new ws(address, { agent: SocksProxyAgent("socks://"+proxySocksAddress) }):new ws(address)
diff --git a/app/modules/ws2p/lib/WS2PServer.ts b/app/modules/ws2p/lib/WS2PServer.ts
index c9d3dc9fb..ae79815fe 100644
--- a/app/modules/ws2p/lib/WS2PServer.ts
+++ b/app/modules/ws2p/lib/WS2PServer.ts
@@ -63,16 +63,23 @@ export class WS2PServer extends events.EventEmitter {
}
return await this.shouldAcceptConnection(pub, this.getConnexions().map(c => c.pubkey))
}
+ let timeout = {
+ connectionTimeout: WS2PConstants.CONNEXION_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TIMEOUT
+ }
+ if (this.server.conf.ws2p && this.server.conf.ws2p.remotehost && this.server.conf.ws2p.remotehost.match(WS2PConstants.HOST_ONION_REGEX)) {
+ timeout = {
+ connectionTimeout: WS2PConstants.CONNEXION_TOR_TIMEOUT,
+ requestTimeout: WS2PConstants.REQUEST_TOR_TIMEOUT
+ }
+ }
const c = WS2PConnection.newConnectionFromWebSocketServer(
ws,
messageHandler,
new WS2PPubkeyLocalAuth(this.server.conf.currency, key, acceptPubkey),
new WS2PPubkeyRemoteAuth(this.server.conf.currency, key, acceptPubkey),
- {
- connectionTimeout: WS2PConstants.CONNEXION_TIMEOUT,
- requestTimeout: WS2PConstants.REQUEST_TIMEOUT
- }
+ timeout
)
try {
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 6a5e93f24..28df5f5fb 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -6,9 +6,10 @@ export const WS2PConstants = {
WS2P_PORTS_END: 20999,
WS2P_UPNP_INTERVAL: 300,
- CONNEXION_TIMEOUT: 10000,
- REQUEST_TIMEOUT: 10000,
- PROXY_TIMEOUT: 30000,
+ CONNEXION_TIMEOUT: 15000,
+ REQUEST_TIMEOUT: 15000,
+ CONNEXION_TOR_TIMEOUT: 30000,
+ REQUEST_TOR_TIMEOUT: 30000,
RECONNEXION_INTERVAL_IN_SEC: 60 * 10, // 10 minutes
BLOCK_PULLING_INTERVAL: 300 * 2, // 10 minutes
@@ -38,7 +39,8 @@ export const WS2PConstants = {
+ '(' + CommonConstants.FORMATS.POW_PREFIX + ')'
+ '$'),
- HOST_ONION_REGEX: new RegExp('(\S*?\.onion)$'),
+ HOST_ONION_REGEX: new RegExp('^(?:www\.)?([0-9a-z]{16}\.onion)$'),
+ FULL_ADDRESS_ONION_REGEX: new RegExp('^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*'),
HEADS_SPREAD_TIMEOUT: 100 // Wait 100ms before sending a bunch of signed heads
}
\ No newline at end of file
diff --git a/index.ts b/index.ts
index 70b7bd28a..f9e3cf445 100644
--- a/index.ts
+++ b/index.ts
@@ -449,10 +449,10 @@ function commandLineConf(program:any, conf:any = {}) {
port: program.port,
},
proxies: {
- proxySocks: program.proxySocks,
- proxyTor: program.proxyTor,
- torAlways: program.torAlways,
- torMixed: program.torMixed,
+ proxySocks: program.socksProxy,
+ proxyTor: program.torProxy,
+ reachingClearEp: program.reachingClearEp,
+ forceTor: program.forceTor,
rmProxies: program.rmProxies
},
logs: {
@@ -467,20 +467,25 @@ function commandLineConf(program:any, conf:any = {}) {
timeout: program.timeout
};
- // Declare proxyConf
- if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.torAlways || cli.proxies.torMixed || cli.proxies.rmProxies) {
- conf.proxyConf = new ProxiesConf()
+ // Declare and update proxiesConf
+ if (cli.proxies.proxySocks || cli.proxies.proxyTor || cli.proxies.reachingClearEp || cli.proxies.forceTor || cli.proxies.rmProxies) {
+ conf.proxiesConf = new ProxiesConf()
+ if (cli.proxies.proxySocks) conf.proxiesConf.proxySocksAddress = cli.proxies.proxySocks;
+ if (cli.proxies.proxyTor) conf.proxiesConf.proxyTorAddress = cli.proxies.proxyTor;
+ if (cli.proxies.reachingClearEp) {
+ switch (cli.proxies.reachingClearEp) {
+ case 'tor': conf.proxiesConf.reachingClearEp = 'tor'; break;
+ case 'none': conf.proxiesConf.reachingClearEp = 'none'; break;
+ }
+ }
+ if (cli.proxies.forceTor) conf.proxiesConf.forceTor = true
}
- // Update conf
+ // Update the rest of the conf
if (cli.currency) conf.currency = cli.currency;
if (cli.server.port) conf.port = cli.server.port;
if (cli.cpu) conf.cpu = Math.max(0.01, Math.min(1.0, cli.cpu));
if (cli.prefix) conf.prefix = Math.max(Constants.MIN_PEER_ID, Math.min(Constants.MAX_PEER_ID, cli.prefix));
- if (cli.proxies.proxySocks && conf.proxyConf) conf.proxyConf.proxySocksAddress = cli.proxies.proxySocks;
- if (cli.proxies.proxyTor && conf.proxyConf) conf.proxyConf.proxyTorAddress = cli.proxies.proxyTor;
- if (cli.proxies.torAlways && conf.proxyConf) conf.proxyConf.alwaysUseTor = true;
- if (cli.proxies.torMixed && conf.proxyConf) conf.proxyConf.alwaysUseTor = false;
if (cli.logs.http) conf.httplogs = true;
if (cli.logs.nohttp) conf.httplogs = false;
if (cli.isolate) conf.isolate = cli.isolate;
diff --git a/server.ts b/server.ts
index 9eca4b855..fff2e8acc 100644
--- a/server.ts
+++ b/server.ts
@@ -149,8 +149,7 @@ export class Server extends stream.Duplex implements HookableServer {
logger.debug('Loading conf...');
this.conf = await this.dal.loadConf(this.overrideConf, useDefaultConf)
// Default values
- this.conf.proxyConf = this.conf.proxyConf === undefined ? new ProxiesConf() : this.conf.proxyConf
- this.conf.proxyConf.alwaysUseTor = this.conf.proxyConf.alwaysUseTor === undefined ? false : this.conf.proxyConf.alwaysUseTor
+ this.conf.proxiesConf = this.conf.proxiesConf === undefined ? new ProxiesConf() : this.conf.proxiesConf
this.conf.remoteipv6 = this.conf.remoteipv6 === undefined ? this.conf.ipv6 : this.conf.remoteipv6
this.conf.remoteport = this.conf.remoteport === undefined ? this.conf.port : this.conf.remoteport
this.conf.c = this.conf.c === undefined ? constants.CONTRACT.DEFAULT.C : this.conf.c
diff --git a/test/fast/proxies.ts b/test/fast/proxies.ts
index c47468092..73377bf21 100644
--- a/test/fast/proxies.ts
+++ b/test/fast/proxies.ts
@@ -13,39 +13,86 @@ describe("Proxies Conf", function() {
// Third conf : always use tor
let proxiesConf3 = new ProxiesConf()
proxiesConf3.proxyTorAddress = "127.0.0.1:9050"
- proxiesConf3.alwaysUseTor = true
+ proxiesConf3.reachingClearEp = 'tor'
- // Fourth cont : use classical socks proxy
+ // Fourth conf : use classical socks proxy
let proxiesConf4 = new ProxiesConf()
proxiesConf4.proxySocksAddress = "127.0.0.1:8888"
- // Fifth : use classical socks proxy + use tor proxy only to reach ".onion" endpoints
+ // Fifth conf : use classical socks proxy + use tor proxy only to reach ".onion" endpoints
let proxiesConf5 = new ProxiesConf()
proxiesConf5.proxySocksAddress = "127.0.0.1:8888"
proxiesConf5.proxyTorAddress = "127.0.0.1:9050"
+ // Sixth conf : always use tor and contact only tor endpoints
+ let proxiesConf6 = new ProxiesConf()
+ proxiesConf6.proxyTorAddress = "127.0.0.1:9050"
+ proxiesConf6.reachingClearEp = 'none'
+
+ // Seventh conf : force duniter to contact endpoint tor (if user redirect the traffic to tor himself)
+ let proxiesConf7 = new ProxiesConf()
+ proxiesConf7.forceTor = true;
+
it('should do not use any sock proxy', () => {
- assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf1) === undefined, true)
- assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf1) === undefined, true)
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf1), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf1), false)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf1), undefined)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf1), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf1), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf1), undefined)
})
it('should use tor proxy only to reach ".onion" endpoints', () => {
- assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf2) === proxiesConf2.proxyTorAddress, true)
- assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf2) === undefined, true)
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf2), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf2), true)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf2), proxiesConf2.proxyTorAddress)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf2), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf2), proxiesConf2.proxyTorAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf2), undefined)
})
it('should always use tor proxy', () => {
- assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf3) === proxiesConf3.proxyTorAddress, true)
- assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf3) === proxiesConf3.proxyTorAddress, true)
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf3), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf3), true)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf3), proxiesConf3.proxyTorAddress)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf3), proxiesConf3.proxyTorAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf3), proxiesConf3.proxyTorAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf3), proxiesConf3.proxyTorAddress)
})
it('should always use classical socks proxy', () => {
- assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf4) === proxiesConf4.proxySocksAddress, true)
- assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf4) === proxiesConf4.proxySocksAddress, true)
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf4), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf4), false)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf4), undefined)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf4), proxiesConf4.proxySocksAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf4), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf4), proxiesConf4.proxySocksAddress)
+ })
+
+ it('should use tor proxy for ".onion" endpoints and classical socks proxy for everyone else', () => {
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf5), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf5), true)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf5), proxiesConf5.proxyTorAddress)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf5), proxiesConf5.proxySocksAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf5), proxiesConf5.proxyTorAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf5), proxiesConf5.proxySocksAddress)
+ })
+
+ it('should always use tor proxy and contact only tor endpoints', () => {
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf6), false)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf6), true)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf6), proxiesConf6.proxyTorAddress)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf6), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf6), proxiesConf6.proxyTorAddress)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf6), undefined)
})
- it('should use or tor proxy for ".onion" endpoints and classical socks proxy for everyone else', () => {
- assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf5) === proxiesConf5.proxyTorAddress, true)
- assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf5) === proxiesConf5.proxySocksAddress, true)
+ it('should never use proxy and contact tor endpoints (user redirect the traffic to tor himself)', () => {
+ assert.equal(ProxiesConf.canReachClearEndpoint(proxiesConf7), true)
+ assert.equal(ProxiesConf.canReachTorEndpoint(proxiesConf7), true)
+ assert.equal(ProxiesConf.httpProxy("3asufnydqmup533h.onion", proxiesConf7), undefined)
+ assert.equal(ProxiesConf.httpProxy("domain.tld", proxiesConf7), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://3asufnydqmup533h.onion:80", proxiesConf7), undefined)
+ assert.equal(ProxiesConf.wsProxy("ws://domain.tld:20900", proxiesConf7), undefined)
})
});
--
GitLab
From 15d534149e5c5089addcd27576b1e82575d8445b Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 16:20:57 +0100
Subject: [PATCH 25/34] [enh] add option to use bma without crawler #1174
---
app/lib/dto/ConfDTO.ts | 4 +++-
app/modules/bma/index.ts | 7 +++++++
app/modules/bma/lib/network.ts | 1 +
app/modules/crawler/lib/crawler.ts | 4 ++--
app/modules/router.ts | 4 ++--
5 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index 1227913e3..f6c57cc5e 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -49,6 +49,7 @@ export interface KeypairConfDTO {
export interface NetworkConfDTO {
proxiesConf: ProxiesConf|undefined
nobma: boolean
+ bmaWithCrawler: boolean
remoteport: number
remotehost: string|null
remoteipv4: string|null
@@ -136,6 +137,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public homename: string,
public memory: boolean,
public nobma: boolean,
+ public bmaWithCrawler: boolean,
public proxiesConf: ProxiesConf|undefined,
public ws2p?: {
privateAccess: boolean
@@ -155,7 +157,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
) {}
static mock() {
- return new ConfDTO("", "", [], [], 0, 0, 0.6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, 0, false, 0, 0, 0, 0, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", 0, "", "", null, false, "", true, true, undefined)
+ return new ConfDTO("", "", [], [], 0, 0, 0.6, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, false, 0, false, 0, 0, 0, 0, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", 0, "", "", null, false, "", true, false, true, undefined)
}
static defaultConf() {
diff --git a/app/modules/bma/index.ts b/app/modules/bma/index.ts
index 914a6ecf4..2abe0ba7a 100644
--- a/app/modules/bma/index.ts
+++ b/app/modules/bma/index.ts
@@ -29,6 +29,8 @@ export const BmaDependency = {
{ value: '--noupnp', desc: 'Do not use UPnP to open remote port.' },
{ value: '--bma', desc: 'Enables BMA API and its crawlers.' },
{ value: '--nobma', desc: 'Disables BMA API and its crawlers.' },
+ { value: '--bma-with-crawler', desc: 'Enables BMA Crawler.' },
+ { value: '--bma-without-crawler', desc: 'Disable BMA Crawler.' },
{ value: '-p, --port <port>', desc: 'Port to listen for requests', parser: (val:string) => parseInt(val) },
{ value: '--ipv4 <address>', desc: 'IPv4 interface to listen for requests' },
{ value: '--ipv6 <address>', desc: 'IPv6 interface to listen for requests' },
@@ -73,6 +75,9 @@ export const BmaDependency = {
}
}
+ // If bmaWithCrawler hasn't been defined yet
+ if (conf.bmaWithCrawler === undefined) { conf.bmaWithCrawler = false }
+
if (program.port !== undefined) conf.port = parseInt(program.port)
if (program.ipv4 !== undefined) conf.ipv4 = program.ipv4;
if (program.ipv6 !== undefined) conf.ipv6 = program.ipv6;
@@ -82,6 +87,8 @@ export const BmaDependency = {
if (program.remotep !== undefined) conf.remoteport = parseInt(program.remotep)
if (program.bma !== undefined) conf.nobma = false
if (program.nobma !== undefined) conf.nobma = true
+ if (program.bmaWithCrawler !== undefined) conf.bmaWithCrawler = true
+ if (program.bmaWithoutCrawler !== undefined) conf.bmaWithCrawler = false
if (!conf.ipv4) delete conf.ipv4;
if (!conf.ipv6) delete conf.ipv6;
diff --git a/app/modules/bma/lib/network.ts b/app/modules/bma/lib/network.ts
index 250a622bf..3e9e026f9 100644
--- a/app/modules/bma/lib/network.ts
+++ b/app/modules/bma/lib/network.ts
@@ -339,6 +339,7 @@ async function upnpConf (noupnp:boolean, logger:any) {
const conf:NetworkConfDTO = {
proxiesConf: undefined,
nobma: true,
+ bmaWithCrawler: false,
port: privatePort,
ipv4: '127.0.0.1',
ipv6: '::1',
diff --git a/app/modules/crawler/lib/crawler.ts b/app/modules/crawler/lib/crawler.ts
index 44bf5a217..9520c5e3a 100644
--- a/app/modules/crawler/lib/crawler.ts
+++ b/app/modules/crawler/lib/crawler.ts
@@ -48,7 +48,7 @@ export class Crawler extends stream.Transform implements DuniterService {
}
startService() {
- if (this.conf.nobma) {
+ if (this.conf.nobma || !this.conf.bmaWithCrawler) {
return Promise.resolve()
}
return Promise.all([
@@ -60,7 +60,7 @@ export class Crawler extends stream.Transform implements DuniterService {
}
stopService() {
- if (this.conf.nobma) {
+ if (this.conf.nobma || !this.conf.bmaWithCrawler) {
return Promise.resolve()
}
return Promise.all([
diff --git a/app/modules/router.ts b/app/modules/router.ts
index 1530cc43e..d6484f05b 100644
--- a/app/modules/router.ts
+++ b/app/modules/router.ts
@@ -44,7 +44,7 @@ class Router extends stream.Transform {
};
async startService() {
- if (this.server.conf.nobma) {
+ if (this.server.conf.nobma || !this.server.conf.bmaWithCrawler) {
// Disable BMA
return Promise.resolve()
}
@@ -69,7 +69,7 @@ class Router extends stream.Transform {
}
async stopService() {
- if (this.server.conf.nobma) {
+ if (this.server.conf.nobma || !this.server.conf.bmaWithCrawler) {
// Disable BMA
return Promise.resolve()
}
--
GitLab
From da51665ce8a0a42d29238f73c60d14f7adbeda2b Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 16:38:55 +0100
Subject: [PATCH 26/34] [enh] create automatically BMATOR endpoint
---
app/lib/common-libs/constants.ts | 6 ++++++
app/lib/proxy.ts | 2 ++
app/modules/bma/index.ts | 3 +++
app/modules/bma/lib/constants.ts | 2 ++
app/modules/ws2p/lib/constants.ts | 4 ++--
5 files changed, 15 insertions(+), 2 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index e44639342..98623e305 100644
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -31,10 +31,13 @@ const UNLOCK = "(SIG\\(" + INTEGER + "\\)|XHX\\(" + XUNLOCK + "\\))"
const CONDITIONS = "(&&|\\|\\|| |[()]|(SIG\\(" + PUBKEY + "\\)|(XHX\\([A-F0-9]{64}\\)|CLTV\\(" + CLTV_INTEGER + "\\)|CSV\\(" + CSV_INTEGER + "\\))))*"
const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
+const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
const WS2P_REGEXP = /^WS2P ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+.onion) ([0-9]+)(?: (.+))?$/
+const WS_FULL_ADDRESS_ONION_REGEX = /'^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*/
const IPV4_REGEXP = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IPV6_REGEXP = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
+const HOST_ONION_REGEX = /'^(?:www\.)?([0-9a-z]{16}\.onion)$'/
const MAXIMUM_LEN_OF_COMPACT_TX = 100
const MAXIMUM_LEN_OF_OUTPUT = 2000
@@ -90,10 +93,13 @@ export const CommonConstants = {
SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS: 3,
BMA_REGEXP,
+ BMATOR_REGEXP,
WS2P_REGEXP,
WS2PTOR_REGEXP,
+ WS_FULL_ADDRESS_ONION_REGEX,
IPV4_REGEXP,
IPV6_REGEXP,
+ HOST_ONION_REGEX,
PUBLIC_KEY: exact(PUBKEY),
INTEGER: /^\d+$/,
BASE58: exact(BASE58),
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index 43b000a22..7df0703de 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -1,3 +1,5 @@
+import {CommonConstants} from "./common-libs/constants"
+
const SocksProxyAgent = require('socks-proxy-agent');
const HOST_ONION_REGEX = new RegExp('(?:www\.)?([0-9a-z]{16}?\.onion)$');
diff --git a/app/modules/bma/index.ts b/app/modules/bma/index.ts
index 2abe0ba7a..d9b4c2d02 100644
--- a/app/modules/bma/index.ts
+++ b/app/modules/bma/index.ts
@@ -260,6 +260,9 @@ export class BMAPI extends stream.Transform {
function getEndpoint(theConf:NetworkConfDTO) {
let endpoint = 'BASIC_MERKLED_API';
if (theConf.remotehost) {
+ if (theConf.remotehost.match(BMAConstants.HOST_ONION_REGEX)) {
+ endpoint = 'BMATOR';
+ }
endpoint += ' ' + theConf.remotehost;
}
if (theConf.remoteipv4) {
diff --git a/app/modules/bma/lib/constants.ts b/app/modules/bma/lib/constants.ts
index 6caa2204b..20a76a6eb 100644
--- a/app/modules/bma/lib/constants.ts
+++ b/app/modules/bma/lib/constants.ts
@@ -1,3 +1,4 @@
+import {CommonConstants} from "../../../lib/common-libs/constants"
export const BMAConstants = {
BMA_PORTS_START: 10901,
@@ -6,6 +7,7 @@ export const BMAConstants = {
DEFAULT_PORT: 10901,
IPV4_REGEXP: /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/,
IPV6_REGEXP: /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/,
+ HOST_ONION_REGEX: CommonConstants.HOST_ONION_REGEX,
PORT_START: 15000,
UPNP_INTERVAL: 300,
UPNP_TTL: 600,
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 28df5f5fb..1607a537c 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -39,8 +39,8 @@ export const WS2PConstants = {
+ '(' + CommonConstants.FORMATS.POW_PREFIX + ')'
+ '$'),
- HOST_ONION_REGEX: new RegExp('^(?:www\.)?([0-9a-z]{16}\.onion)$'),
- FULL_ADDRESS_ONION_REGEX: new RegExp('^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*'),
+ HOST_ONION_REGEX: CommonConstants.HOST_ONION_REGEX,
+ FULL_ADDRESS_ONION_REGEX: CommonConstants.WS_FULL_ADDRESS_ONION_REGEX,
HEADS_SPREAD_TIMEOUT: 100 // Wait 100ms before sending a bunch of signed heads
}
\ No newline at end of file
--
GitLab
From cef5096e5886ecdcadb060ab664a3d060dc8fe32 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 16:45:26 +0100
Subject: [PATCH 27/34] [fix] enable bma crawler for test integration
---
test/integration/tools/toolbox.ts | 1 +
1 file changed, 1 insertion(+)
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index 85c317dff..d0db935b4 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -226,6 +226,7 @@ export const NewTestingServer = (conf:any) => {
const port = conf.port ||Â PORT++
const commonConf = {
nobma: false,
+ bmaWithCrawler: true,
port: port,
ipv4: host,
remoteipv4: host,
--
GitLab
From a0b0f3ac672761c02fd1073aabae76b74f8f4b7c Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 17:02:48 +0100
Subject: [PATCH 28/34] [fix] repair integration tests forwarding and network
---
test/integration/forwarding.js | 2 +-
test/integration/network.js | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/test/integration/forwarding.js b/test/integration/forwarding.js
index b550a400b..ba392a69a 100644
--- a/test/integration/forwarding.js
+++ b/test/integration/forwarding.js
@@ -19,7 +19,7 @@ describe("Forwarding", function() {
describe("Nodes", function() {
- const common = { currency: 'bb', nobma: false, ws2p: { upnp: false }, ipv4: '127.0.0.1', remoteipv4: '127.0.0.1', rootoffset: 0, sigQty: 1 };
+ const common = { currency: 'bb', nobma: false, bmaWithCrawler:true, ws2p: { upnp: false }, ipv4: '127.0.0.1', remoteipv4: '127.0.0.1', rootoffset: 0, sigQty: 1 };
const node1 = node('db_1', _({ upnp: false, httplogs: false, port: 9600, remoteport: 9600, pair: { pub: 'HgTTJLAQ5sqfknMq7yLPZbehtuLSsKj9CxWN7k8QvYJd', sec: '51w4fEShBk1jCMauWu4mLpmDVfHksKmWcygpxriqCEZizbtERA6de4STKRkQBpxmMUwsKXRjSzuQ8ECwmqN1u2DP'} }).extend(common));
const node2 = node('db_2', _({ upnp: false, httplogs: false, port: 9601, remoteport: 9601, pair: { pub: 'G2CBgZBPLe6FSFUgpx2Jf1Aqsgta6iib3vmDRA1yLiqU', sec: '58LDg8QLmF5pv6Dn9h7X4yFKfMTdP8fdAiWVcyDoTRJu454fwRihCLULH4MW37zncsg4ruoTGJPZneWk22QmG1w4'} }).extend(common));
diff --git a/test/integration/network.js b/test/integration/network.js
index 32c2fae9e..88a4afa84 100644
--- a/test/integration/network.js
+++ b/test/integration/network.js
@@ -11,6 +11,7 @@ const expectAnswer = httpTest.expectAnswer;
const MEMORY_MODE = true;
const commonConf = {
+ bmaWithCrawler: true,
ipv4: '127.0.0.1',
remoteipv4: '127.0.0.1',
currency: 'bb',
--
GitLab
From 7ec8440d7d23f92e12976f1aa090ad10cdd103df Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 17:47:27 +0100
Subject: [PATCH 29/34] [fix] test start_generate_blocks
---
test/integration/start_generate_blocks.js | 1 +
test/integration/tools/toolbox.ts | 1 +
2 files changed, 2 insertions(+)
diff --git a/test/integration/start_generate_blocks.js b/test/integration/start_generate_blocks.js
index e700d38d5..c71bd8c80 100644
--- a/test/integration/start_generate_blocks.js
+++ b/test/integration/start_generate_blocks.js
@@ -19,6 +19,7 @@ const expectJSON = httpTest.expectJSON;
const MEMORY_MODE = true;
const commonConf = {
+ bmaWithCrawler: true,
ipv4: '127.0.0.1',
remoteipv4: '127.0.0.1',
currency: 'bb',
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index d0db935b4..cbbe1d156 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -697,6 +697,7 @@ export const simpleWS2PNetwork: (s1: TestingServer, s2: TestingServer) => Promis
export function simpleTestingConf(now = 1500000000, pair:{ pub:string, sec:string }): any {
return {
+ bmaWithCrawler: true,
pair,
nbCores: 1,
udTime0: now,
--
GitLab
From 94a394ee713208e128b744d7ad0028f3c5e3e879 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 18:00:35 +0100
Subject: [PATCH 30/34] [fix] test peerings
---
test/integration/peerings.js | 1 +
1 file changed, 1 insertion(+)
diff --git a/test/integration/peerings.js b/test/integration/peerings.js
index b9a729613..da7f16c6e 100644
--- a/test/integration/peerings.js
+++ b/test/integration/peerings.js
@@ -22,6 +22,7 @@ const expectJSON = httpTest.expectJSON;
const MEMORY_MODE = true;
const commonConf = {
+ bmaWithCrawler: true,
ipv4: '127.0.0.1',
remoteipv4: '127.0.0.1',
currency: 'bb',
--
GitLab
From 40e3c3a7de5fce3eae19a01ea42e2dbb9eadd6f0 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 21:44:01 +0100
Subject: [PATCH 31/34] [fix] regex HOST_ONION_REGEX (and increase
REQUEST_TOR_TIMEOUT)
---
app/lib/common-libs/constants.ts | 2 +-
app/modules/ws2p/lib/constants.ts | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 98623e305..d67a34036 100644
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -37,7 +37,7 @@ const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+.
const WS_FULL_ADDRESS_ONION_REGEX = /'^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*/
const IPV4_REGEXP = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IPV6_REGEXP = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
-const HOST_ONION_REGEX = /'^(?:www\.)?([0-9a-z]{16}\.onion)$'/
+const HOST_ONION_REGEX = /^(?:www\.)?([0-9a-z]{16}\.onion)$/
const MAXIMUM_LEN_OF_COMPACT_TX = 100
const MAXIMUM_LEN_OF_OUTPUT = 2000
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 1607a537c..ca457f1a9 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -9,7 +9,7 @@ export const WS2PConstants = {
CONNEXION_TIMEOUT: 15000,
REQUEST_TIMEOUT: 15000,
CONNEXION_TOR_TIMEOUT: 30000,
- REQUEST_TOR_TIMEOUT: 30000,
+ REQUEST_TOR_TIMEOUT: 60000,
RECONNEXION_INTERVAL_IN_SEC: 60 * 10, // 10 minutes
BLOCK_PULLING_INTERVAL: 300 * 2, // 10 minutes
--
GitLab
From 5e4997b5ef4f1ad569110835e5cb7dbb3b252a9e Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Fri, 3 Nov 2017 21:48:48 +0100
Subject: [PATCH 32/34] [fix] wrong format WS_FULL_ADDRESS_ONION_REGEX
---
app/lib/common-libs/constants.ts | 2 +-
app/lib/proxy.ts | 7 ++-----
2 files changed, 3 insertions(+), 6 deletions(-)
diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index d67a34036..89b6e7dba 100644
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -34,7 +34,7 @@ const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([
const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/
const WS2P_REGEXP = /^WS2P ([a-f0-9]{8}) ([a-z_][a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+) ([0-9]+)(?: (.+))?$/
const WS2PTOR_REGEXP = /^WS2PTOR ([a-f0-9]{8}) ([a-z0-9-_.]*|[0-9.]+|[0-9a-f:]+.onion) ([0-9]+)(?: (.+))?$/
-const WS_FULL_ADDRESS_ONION_REGEX = /'^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*/
+const WS_FULL_ADDRESS_ONION_REGEX = /^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?$/
const IPV4_REGEXP = /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/;
const IPV6_REGEXP = /^((([0-9A-Fa-f]{1,4}:){7}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}:[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){5}:([0-9A-Fa-f]{1,4}:)?[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){4}:([0-9A-Fa-f]{1,4}:){0,2}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){3}:([0-9A-Fa-f]{1,4}:){0,3}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){2}:([0-9A-Fa-f]{1,4}:){0,4}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){6}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(([0-9A-Fa-f]{1,4}:){0,5}:((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|(::([0-9A-Fa-f]{1,4}:){0,5}((b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b).){3}(b((25[0-5])|(1d{2})|(2[0-4]d)|(d{1,2}))b))|([0-9A-Fa-f]{1,4}::([0-9A-Fa-f]{1,4}:){0,5}[0-9A-Fa-f]{1,4})|(::([0-9A-Fa-f]{1,4}:){0,6}[0-9A-Fa-f]{1,4})|(([0-9A-Fa-f]{1,4}:){1,7}:))$/;
const HOST_ONION_REGEX = /^(?:www\.)?([0-9a-z]{16}\.onion)$/
diff --git a/app/lib/proxy.ts b/app/lib/proxy.ts
index 7df0703de..340de605c 100644
--- a/app/lib/proxy.ts
+++ b/app/lib/proxy.ts
@@ -2,9 +2,6 @@ import {CommonConstants} from "./common-libs/constants"
const SocksProxyAgent = require('socks-proxy-agent');
-const HOST_ONION_REGEX = new RegExp('(?:www\.)?([0-9a-z]{16}?\.onion)$');
-const WS_FULL_ADDRESS_ONION_REGEX = new RegExp('^(?:wss?:\/\/)(?:www\.)?([0-9a-z]{16}\.onion)(:[0-9]+)?(\/[-\w]*)*');
-
export class ProxiesConf {
public proxySocksAddress: string|undefined
public proxyTorAddress: string|undefined
@@ -27,11 +24,11 @@ export class ProxiesConf {
}
static httpProxy(url:string, proxiesConf: ProxiesConf|undefined):string|undefined {
- return ProxiesConf.chooseProxyAgent(url, proxiesConf, HOST_ONION_REGEX)
+ return ProxiesConf.chooseProxyAgent(url, proxiesConf, CommonConstants.HOST_ONION_REGEX)
}
static wsProxy(address:string, proxiesConf: ProxiesConf|undefined):string|undefined {
- return ProxiesConf.chooseProxyAgent(address, proxiesConf, WS_FULL_ADDRESS_ONION_REGEX)
+ return ProxiesConf.chooseProxyAgent(address, proxiesConf, CommonConstants.WS_FULL_ADDRESS_ONION_REGEX)
}
private static chooseProxyAgent(address:string, proxiesConf: ProxiesConf|undefined, onionRegex:RegExp):string|undefined {
--
GitLab
From 8c33efe2fe7f7e2a706be0feccbc633e51e273e4 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sat, 4 Nov 2017 01:45:27 +0100
Subject: [PATCH 33/34] [enh] prepare api type in head mesage
---
app/modules/ws2p/lib/WS2PCluster.ts | 5 +++--
app/modules/ws2p/lib/constants.ts | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/app/modules/ws2p/lib/WS2PCluster.ts b/app/modules/ws2p/lib/WS2PCluster.ts
index 875c552dd..701973334 100644
--- a/app/modules/ws2p/lib/WS2PCluster.ts
+++ b/app/modules/ws2p/lib/WS2PCluster.ts
@@ -291,7 +291,7 @@ export class WS2PCluster {
async connectToRemoteWS(host: string, port: number, path:string, messageHandler:WS2PMessageHandler, expectedPub:string, ws2pEndpointUUID:string = ""): Promise<WS2PConnection> {
const uuid = nuuid.v4()
- let pub = "--------"
+ let pub = expectedPub.slice(0, 8)
const api:string = (host.match(WS2PConstants.HOST_ONION_REGEX) !== null) ? 'WS2PTOR':'WS2P'
try {
const fullEndpointAddress = WS2PCluster.getFullAddress(host, port, path)
@@ -463,13 +463,14 @@ export class WS2PCluster {
}
private sayHeadChangedTo(number:number, hash:string) {
+ const api = (this.server.conf.ws2p && this.server.conf.ws2p.remotehost && this.server.conf.ws2p.remotehost.match(WS2PConstants.HOST_ONION_REGEX)) ? 'WS2P':'WS2P'
const key = new Key(this.server.conf.pair.pub, this.server.conf.pair.sec)
const pub = key.publicKey
const software = 'duniter'
const softVersion = Package.getInstance().version
const ws2pId = (this.server.conf.ws2p && this.server.conf.ws2p.uuid) || '00000000'
const prefix = this.server.conf.prefix || Constants.DEFAULT_PEER_ID
- const message = `WS2P:HEAD:1:${pub}:${number}-${hash}:${ws2pId}:${software}:${softVersion}:${prefix}`
+ const message = `${api}:HEAD:1:${pub}:${number}-${hash}:${ws2pId}:${software}:${softVersion}:${prefix}`
const sig = key.signSync(message)
return { sig, message, pub }
}
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index ca457f1a9..8338f8dbf 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -30,7 +30,7 @@ export const WS2PConstants = {
+ CommonConstants.FORMATS.BLOCKSTAMP
+ '$'),
- HEAD_V1_REGEXP: new RegExp('^WS2P:HEAD:1:'
+ HEAD_V1_REGEXP: new RegExp('^WS2P(?:TOR)?:HEAD:1:'
+ '(' + CommonConstants.FORMATS.PUBKEY + '):'
+ '(' + CommonConstants.FORMATS.BLOCKSTAMP + '):'
+ '(' + CommonConstants.FORMATS.WS2PID + '):'
--
GitLab
From b3100e4bbaa7174bc35f0a99e33c9d308bac7f37 Mon Sep 17 00:00:00 2001
From: librelois <elois@ifee.fr>
Date: Sat, 4 Nov 2017 18:05:44 +0100
Subject: [PATCH 34/34] prepare api type details in head mesage
---
app/modules/ws2p/lib/constants.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/app/modules/ws2p/lib/constants.ts b/app/modules/ws2p/lib/constants.ts
index 8338f8dbf..51972d7f7 100644
--- a/app/modules/ws2p/lib/constants.ts
+++ b/app/modules/ws2p/lib/constants.ts
@@ -30,7 +30,7 @@ export const WS2PConstants = {
+ CommonConstants.FORMATS.BLOCKSTAMP
+ '$'),
- HEAD_V1_REGEXP: new RegExp('^WS2P(?:TOR)?:HEAD:1:'
+ HEAD_V1_REGEXP: new RegExp('^WS2P(?:O[CT][SAM])?(?:I[CT])?:HEAD:1:'
+ '(' + CommonConstants.FORMATS.PUBKEY + '):'
+ '(' + CommonConstants.FORMATS.BLOCKSTAMP + '):'
+ '(' + CommonConstants.FORMATS.WS2PID + '):'
--
GitLab