diff --git a/app/lib/common-libs/constants.ts b/app/lib/common-libs/constants.ts
index 3b16ff3e87429ced051393a61703a58f250746cb..664cecf585a0edf56cebe60e380ee729408b9c18 100755
--- a/app/lib/common-libs/constants.ts
+++ b/app/lib/common-libs/constants.ts
@@ -309,6 +309,7 @@ export const CommonConstants = {
INITIAL_DOWNLOAD_SLOTS: 1, // 1 peer
BLOCKS_COLLECT_THRESHOLD: 30, // Number of blocks to wait before trimming the loki data
+ DEFAULT_NON_WOT_PEERS_LIMIT: 100, // Number of non-wot peers accepted in our peer document pool
}
function exact (regexpContent:string) {
diff --git a/app/lib/common-libs/errors.ts b/app/lib/common-libs/errors.ts
index ffaa416c8ce74aeed8426d044402354fdc43f33f..b01172a5a8760de8c2ddb2ec33acacec181dd001 100755
--- a/app/lib/common-libs/errors.ts
+++ b/app/lib/common-libs/errors.ts
@@ -1,5 +1,6 @@
export enum DataErrors {
+ PEER_REJECTED,
TOO_OLD_PEER,
LOKI_DIVIDEND_GET_WRITTEN_ON_SHOULD_NOT_BE_USED,
LOKI_DIVIDEND_REMOVE_BLOCK_SHOULD_NOT_BE_USED,
diff --git a/app/lib/dal/indexDAL/abstract/PeerDAO.ts b/app/lib/dal/indexDAL/abstract/PeerDAO.ts
index 97756b4227a35787eccd1b955568acca7069d98c..5a574c1c6d138473e8387bf112e97a9f82266adf 100644
--- a/app/lib/dal/indexDAL/abstract/PeerDAO.ts
+++ b/app/lib/dal/indexDAL/abstract/PeerDAO.ts
@@ -48,15 +48,21 @@ export interface PeerDAO extends Initiable, LokiDAO {
removePeerByPubkey(pubkey:string): Promise<void>
/**
- * Remove peers that were set down before a certain datetime.
- * @param {number} thresholdTime
+ * Remove all the peers.
* @returns {Promise<void>}
*/
- removePeersDownBefore(thresholdTime:number): Promise<void>
+ removeAll(): Promise<void>
/**
- * Remove all the peers.
+ * Count the number of non-WoT peers known is the DB.
+ * @returns {Promise<number>} The number of nonWoT peers.
+ */
+ countNonWoTPeers(): Promise<number>
+
+ /**
+ * Remove all **non-WoT** peers whose last contact is above given time (timestamp in seconds).
+ * @param {number} threshold
* @returns {Promise<void>}
*/
- removeAll(): Promise<void>
+ deletePeersWhoseLastContactIsAbove(threshold: number): Promise<void>
}
diff --git a/app/lib/dal/indexDAL/loki/LokiPeer.ts b/app/lib/dal/indexDAL/loki/LokiPeer.ts
index c067aa2976b1379e0b8548677a4c44f0f329b2d8..aa2b22b8ccbbb80f6d73740df658715c30384d45 100644
--- a/app/lib/dal/indexDAL/loki/LokiPeer.ts
+++ b/app/lib/dal/indexDAL/loki/LokiPeer.ts
@@ -5,7 +5,7 @@ import {DBPeer} from "../../../db/DBPeer"
export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
constructor(loki:any) {
- super(loki, 'peer', ['pubkey'])
+ super(loki, 'peer', ['pubkey', 'nonWoT', 'lastContact'])
}
async init(): Promise<void> {
@@ -55,6 +55,8 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
p.signature = peer.signature
p.endpoints = peer.endpoints
p.raw = peer.raw
+ p.nonWoT = peer.nonWoT
+ p.lastContact = peer.lastContact
updated = true
})
if (!updated) {
@@ -70,18 +72,6 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
.remove()
}
- async removePeersDownBefore(thresholdTime:number): Promise<void> {
- this.collection
- .chain()
- .find({
- $and: [
- { first_down: { $lt: thresholdTime } },
- { first_down: { $gt: 0 } },
- ]
- })
- .remove()
- }
-
async removeAll(): Promise<void> {
this.collection
.chain()
@@ -104,4 +94,22 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
.where(p => p.endpoints.filter(ep => ep.indexOf(ep) !== -1).length > 0)
.data()
}
+
+ async countNonWoTPeers(): Promise<number> {
+ return this.collection
+ .find({ nonWoT: true })
+ .length
+ }
+
+ async deletePeersWhoseLastContactIsAbove(threshold: number) {
+ this.collection
+ .chain()
+ .find({
+ $or: [
+ { lastContact: { $lt: threshold } },
+ { lastContact: null },
+ ]
+ })
+ .remove()
+ }
}
\ No newline at end of file
diff --git a/app/lib/db/DBPeer.ts b/app/lib/db/DBPeer.ts
index c71fb86e76500e80ecad0184c121606bd762e09e..e4bbada5d9ac61ad12dfcc1bf1a23fcd71e26673 100644
--- a/app/lib/db/DBPeer.ts
+++ b/app/lib/db/DBPeer.ts
@@ -9,11 +9,13 @@ export class DBPeer {
hash: string
first_down: number | null
last_try: number | null
+ lastContact: number = Math.floor(Date.now() / 1000)
pubkey: string
block: string
signature: string
endpoints: string[]
raw: string
+ nonWoT: boolean = true // Security measure: a peer is presumed nonWoT.
static json(peer:DBPeer): JSONDBPeer {
return {
@@ -25,7 +27,7 @@ export class DBPeer {
pubkey: peer.pubkey,
block: peer.block,
signature: peer.signature,
- endpoints: peer.endpoints
+ endpoints: peer.endpoints,
}
}
diff --git a/app/lib/dto/ConfDTO.ts b/app/lib/dto/ConfDTO.ts
index a95feec6ae550a2be1290de75d47bdef7d8cceb5..c8075acaf9813938fb59bd6ebe759bbcb83bd59b 100644
--- a/app/lib/dto/ConfDTO.ts
+++ b/app/lib/dto/ConfDTO.ts
@@ -78,6 +78,7 @@ export interface NetworkConfDTO {
dos:any
upnp:boolean
httplogs:boolean
+ nonWoTPeersLimit: number
}
export interface WS2PConfDTO {
@@ -159,6 +160,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public memory: boolean,
public nobma: boolean,
public bmaWithCrawler: boolean,
+ public nonWoTPeersLimit: number,
public proxiesConf: ProxiesConf|undefined,
public ws2p?: {
privateAccess?: boolean
@@ -181,7 +183,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
) {}
static mock() {
- return new ConfDTO("", "", [], [], 0, 3600 * 1000, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, new ProxiesConf(), undefined)
+ return new ConfDTO("", "", [], [], 0, 3600 * 1000, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, 100, new ProxiesConf(), undefined)
}
static defaultConf() {
@@ -211,7 +213,8 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
"timeout": 3000,
"isolate": false,
"forksize": constants.BRANCHES.DEFAULT_WINDOW_SIZE,
- "switchOnHeadAdvance": CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS
+ "switchOnHeadAdvance": CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS,
+ "nonWoTPeersLimit": CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT,
};
}
diff --git a/app/modules/bma/lib/network.ts b/app/modules/bma/lib/network.ts
index c75d8d79451a7db6dbc1ca7cccd24861c338a673..ba36638287c5b4dc7fd47ec7d4454ccc1e0cf052 100644
--- a/app/modules/bma/lib/network.ts
+++ b/app/modules/bma/lib/network.ts
@@ -16,6 +16,7 @@ import {Server} from "../../../../server"
import {BMAConstants} from "./constants"
import {BMALimitation} from "./limiter"
import {Underscore} from "../../../lib/common-libs/underscore"
+import {CommonConstants} from "../../../lib/common-libs/constants"
const os = require('os');
const Q = require('q');
@@ -361,7 +362,8 @@ async function upnpConf (noupnp:boolean, logger:any) {
remoteport: publicPort,
remotehost: null,
remoteipv4: null,
- remoteipv6: null
+ remoteipv6: null,
+ nonWoTPeersLimit: CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT
}
logger && logger.info('Checking UPnP features...');
if (noupnp) {
diff --git a/app/modules/crawler/index.ts b/app/modules/crawler/index.ts
index 2d72d7b411e6751aa1d118c79c3178925db994ed..c2b5ffe7e5456d8ab2329d833c69c03666abea5d 100644
--- a/app/modules/crawler/index.ts
+++ b/app/modules/crawler/index.ts
@@ -150,7 +150,7 @@ export const CrawlerDependency = {
logger.info('Fetching peering record at %s:%s...', host, port);
let peering = await Contacter.fetchPeer(host, port);
logger.info('Apply peering ...');
- await server.PeeringService.submitP(peering, ERASE_IF_ALREADY_RECORDED, !program.nocautious);
+ await server.PeeringService.submitP(peering, ERASE_IF_ALREADY_RECORDED, !program.nocautious, true);
logger.info('Applied');
let selfPeer = await server.dal.getPeer(server.PeeringService.pubkey);
if (!selfPeer) {
diff --git a/app/modules/crawler/lib/garbager.ts b/app/modules/crawler/lib/garbager.ts
index 852d33187eb872fe9b768b9c50c12690dae73f09..6c181b58f65c066b34a4a66814cfac04bde80c92 100644
--- a/app/modules/crawler/lib/garbager.ts
+++ b/app/modules/crawler/lib/garbager.ts
@@ -15,6 +15,6 @@ import {CrawlerConstants} from "./constants"
import {Server} from "../../../../server"
export const cleanLongDownPeers = async (server:Server, now:number) => {
- const first_down_limit = now - CrawlerConstants.PEER_LONG_DOWN * 1000;
- await server.dal.peerDAL.removePeersDownBefore(first_down_limit)
+ const first_down_limit = Math.floor((now - CrawlerConstants.PEER_LONG_DOWN * 1000) / 1000)
+ await server.dal.peerDAL.deletePeersWhoseLastContactIsAbove(first_down_limit)
}
diff --git a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
index 8113ec42d4c6a0fcd9ca2ad123e9f0c22e7aaa68..9767f3de60c15beab8c687d68ced2aadd468cb51 100644
--- a/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
+++ b/app/modules/crawler/lib/sync/RemoteSynchronizer.ts
@@ -91,6 +91,8 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
async init(): Promise<void> {
const peering = await Contacter.fetchPeer(this.host, this.port, RemoteSynchronizer.contacterOptions)
this.peer = PeerDTO.fromJSONObject(peering)
+ // We save this peer as a trusted peer for future contact
+ await this.server.PeeringService.submitP(DBPeer.fromPeerDTO(this.peer), false, false, true)
logger.info("Try with %s %s", this.peer.getURL(), this.peer.pubkey.substr(0, 6))
this.node = await connect(this.peer)
;(this.node as any).pubkey = this.peer.pubkey
diff --git a/app/service/PeeringService.ts b/app/service/PeeringService.ts
index 41c6660701257fabc01e3c34db4777d3328845df..6d9b9c6e5033749604a135a73e488402b26e3064 100755
--- a/app/service/PeeringService.ts
+++ b/app/service/PeeringService.ts
@@ -25,6 +25,7 @@ import {DBPeer} from "../lib/db/DBPeer"
import {Underscore} from "../lib/common-libs/underscore"
import {CommonConstants} from "../lib/common-libs/constants"
import {DataErrors} from "../lib/common-libs/errors"
+import {cleanLongDownPeers} from "../modules/crawler/lib/garbager"
const util = require('util');
const events = require('events');
@@ -85,7 +86,7 @@ export class PeeringService {
return !!signaturesMatching;
};
- submitP(peering:DBPeer, eraseIfAlreadyRecorded = false, cautious = true): Promise<PeerDTO> {
+ submitP(peering:DBPeer, eraseIfAlreadyRecorded = false, cautious = true, acceptNonWoT = false): Promise<PeerDTO> {
// Force usage of local currency name, do not accept other currencies documents
peering.currency = this.conf.currency || peering.currency;
let thePeerDTO = PeerDTO.fromJSONObject(peering)
@@ -99,12 +100,42 @@ export class PeeringService {
const hash = thePeerDTO.getHash()
return this.fifoPromiseHandler.pushFIFOPromise<PeerDTO>(hash, async () => {
try {
+ // First: let's make a cleanup of old peers
+ await cleanLongDownPeers(this.server, Date.now())
if (makeCheckings) {
let goodSignature = this.checkPeerSignature(thePeerDTO)
if (!goodSignature) {
throw 'Signature from a peer must match';
}
}
+ // We accept peer documents up to 100 entries, then only member or specific peers are accepted
+ let isNonWoT = false
+ if (!acceptNonWoT) {
+ // Of course we accept our own key
+ if (peering.pubkey !== this.conf.pair.pub) {
+ // As well as prefered/priviledged nodes
+ const isInPrivileged = this.conf.ws2p
+ && this.conf.ws2p.privilegedNodes
+ && this.conf.ws2p.privilegedNodes.length
+ && this.conf.ws2p.privilegedNodes.indexOf(peering.pubkey) !== -1
+ const isInPrefered = this.conf.ws2p
+ && this.conf.ws2p.preferedNodes
+ && this.conf.ws2p.preferedNodes.length
+ && this.conf.ws2p.preferedNodes.indexOf(peering.pubkey) !== -1
+ if (!isInPrefered && !isInPrivileged) {
+ // We also accept all members
+ const isMember = await this.dal.isMember(this.conf.pair.pub)
+ if (!isMember) {
+ isNonWoT = true
+ // Then as long as we have some room, we accept peers
+ const hasEnoughRoom = (await this.dal.peerDAL.countNonWoTPeers()) < this.conf.nonWoTPeersLimit
+ if (!hasEnoughRoom) {
+ throw Error(DataErrors[DataErrors.PEER_REJECTED])
+ }
+ }
+ }
+ }
+ }
if (thePeer.block == constants.PEER.SPECIAL_BLOCK) {
thePeer.block = constants.PEER.SPECIAL_BLOCK;
thePeer.statusTS = 0;
@@ -118,8 +149,8 @@ export class PeeringService {
thePeer.statusTS = 0;
thePeer.status = 'UP';
}
- const current = await this.dal.getBlockCurrent()
- if ((!block && current.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS) || (block && current.number - block.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS)) {
+ const current = await this.dal.getCurrentBlockOrNull()
+ if (current && ((!block && current.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS) || (block && current.number - block.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS))) {
throw Error(DataErrors[DataErrors.TOO_OLD_PEER])
}
}
@@ -167,6 +198,8 @@ export class PeeringService {
peerEntity.last_try = null;
peerEntity.hash = peerEntityOld.getHash()
peerEntity.raw = peerEntityOld.getRaw();
+ peerEntity.nonWoT = isNonWoT
+ peerEntity.lastContact = Math.floor(Date.now() / 1000)
await this.dal.savePeer(peerEntity);
this.logger.info('✔ PEER %s', peering.pubkey.substr(0, 8))
let savedPeer = PeerDTO.fromJSONObject(peerEntity).toDBPeer()
diff --git a/test/fast/modules/crawler/crawler-peers-garbaging.ts b/test/fast/modules/crawler/crawler-peers-garbaging.ts
index 9bb881fc823aa43c9f632bd2e0ad711b68cac808..5f51ce67e5c1d038b09f82f7e3a59c7220888993 100644
--- a/test/fast/modules/crawler/crawler-peers-garbaging.ts
+++ b/test/fast/modules/crawler/crawler-peers-garbaging.ts
@@ -33,14 +33,14 @@ describe('Peers garbaging', () => {
desc: 'Garbage testing',
logs: false,
onDatabaseExecute: async (server:Server) => {
- await server.dal.peerDAL.savePeer({ pubkey: 'A', version: 1, currency: 'c', first_down: null, statusTS: 1485000000000, block: '2393-H' } as any);
- await server.dal.peerDAL.savePeer({ pubkey: 'B', version: 1, currency: 'c', first_down: 1484827199999, statusTS: 1485000000000, block: '2393-H' } as any);
- await server.dal.peerDAL.savePeer({ pubkey: 'C', version: 1, currency: 'c', first_down: 1484827200000, statusTS: 1485000000000, block: '2393-H' } as any);
- await server.dal.peerDAL.savePeer({ pubkey: 'D', version: 1, currency: 'c', first_down: 1484820000000, statusTS: 1485000000000, block: '2393-H' } as any);
+ await server.dal.peerDAL.savePeer({ pubkey: 'A', version: 1, currency: 'c', lastContact: null, statusTS: 1485000000000, block: '2393-H' } as any);
+ await server.dal.peerDAL.savePeer({ pubkey: 'B', version: 1, currency: 'c', lastContact: 1484827199, statusTS: 1485000000000, block: '2393-H' } as any);
+ await server.dal.peerDAL.savePeer({ pubkey: 'C', version: 1, currency: 'c', lastContact: 1484827200, statusTS: 1485000000000, block: '2393-H' } as any);
+ await server.dal.peerDAL.savePeer({ pubkey: 'D', version: 1, currency: 'c', lastContact: 1484820000, statusTS: 1485000000000, block: '2393-H' } as any);
(await server.dal.peerDAL.listAll()).should.have.length(4);
- const now = 1485000000000;
+ const now = 1485000000000
await cleanLongDownPeers(server, now);
- (await server.dal.peerDAL.listAll()).should.have.length(2);
+ (await server.dal.peerDAL.listAll()).should.have.length(1);
}
}]
}
diff --git a/test/integration/tools/toolbox.ts b/test/integration/tools/toolbox.ts
index 677c3b2a86ee976c3ca8406f5613c3d73be95f74..29090c972a26ca4d3d1e9cd88f31160f53337ce2 100644
--- a/test/integration/tools/toolbox.ts
+++ b/test/integration/tools/toolbox.ts
@@ -50,6 +50,7 @@ import {until} from "./test-until"
import {sync} from "./test-sync"
import {expectAnswer, expectError, expectJSON} from "./http-expect"
import {WebSocketServer} from "../../../app/lib/common-libs/websocket"
+import {CommonConstants} from "../../../app/lib/common-libs/constants"
const assert = require('assert');
const rp = require('request-promise');
@@ -228,7 +229,8 @@ export const NewTestingServer = (conf:any) => {
remoteipv4: host,
currency: conf.currency || CURRENCY_NAME,
httpLogs: true,
- forksize: conf.forksize || 3
+ forksize: conf.forksize || 3,
+ nonWoTPeersLimit: CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT,
};
if (conf.sigQty === undefined) {
conf.sigQty = 1;