Mise à jour de GitLab prévue ce samedi 23 octobre 2021 à partir de 9h00 CET

Commit 5b2dc4f6 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

[enh] securiyty: add a `nonWoTPeersLimit` conf parameter to avoid a peers overflow

parent 86ab7796
......@@ -309,6 +309,7 @@ export const CommonConstants = {
INITIAL_DOWNLOAD_SLOTS: 1, // 1 peer
BLOCKS_COLLECT_THRESHOLD: 30, // Number of blocks to wait before trimming the loki data
DEFAULT_NON_WOT_PEERS_LIMIT: 100, // Number of non-wot peers accepted in our peer document pool
}
function exact (regexpContent:string) {
......
export enum DataErrors {
PEER_REJECTED,
TOO_OLD_PEER,
LOKI_DIVIDEND_GET_WRITTEN_ON_SHOULD_NOT_BE_USED,
LOKI_DIVIDEND_REMOVE_BLOCK_SHOULD_NOT_BE_USED,
......
......@@ -48,15 +48,21 @@ export interface PeerDAO extends Initiable, LokiDAO {
removePeerByPubkey(pubkey:string): Promise<void>
/**
* Remove peers that were set down before a certain datetime.
* @param {number} thresholdTime
* Remove all the peers.
* @returns {Promise<void>}
*/
removePeersDownBefore(thresholdTime:number): Promise<void>
removeAll(): Promise<void>
/**
* Remove all the peers.
* Count the number of non-WoT peers known is the DB.
* @returns {Promise<number>} The number of nonWoT peers.
*/
countNonWoTPeers(): Promise<number>
/**
* Remove all **non-WoT** peers whose last contact is above given time (timestamp in seconds).
* @param {number} threshold
* @returns {Promise<void>}
*/
removeAll(): Promise<void>
deletePeersWhoseLastContactIsAbove(threshold: number): Promise<void>
}
......@@ -5,7 +5,7 @@ import {DBPeer} from "../../../db/DBPeer"
export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
constructor(loki:any) {
super(loki, 'peer', ['pubkey'])
super(loki, 'peer', ['pubkey', 'nonWoT', 'lastContact'])
}
async init(): Promise<void> {
......@@ -55,6 +55,8 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
p.signature = peer.signature
p.endpoints = peer.endpoints
p.raw = peer.raw
p.nonWoT = peer.nonWoT
p.lastContact = peer.lastContact
updated = true
})
if (!updated) {
......@@ -70,18 +72,6 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
.remove()
}
async removePeersDownBefore(thresholdTime:number): Promise<void> {
this.collection
.chain()
.find({
$and: [
{ first_down: { $lt: thresholdTime } },
{ first_down: { $gt: 0 } },
]
})
.remove()
}
async removeAll(): Promise<void> {
this.collection
.chain()
......@@ -104,4 +94,22 @@ export class LokiPeer extends LokiCollectionManager<DBPeer> implements PeerDAO {
.where(p => p.endpoints.filter(ep => ep.indexOf(ep) !== -1).length > 0)
.data()
}
async countNonWoTPeers(): Promise<number> {
return this.collection
.find({ nonWoT: true })
.length
}
async deletePeersWhoseLastContactIsAbove(threshold: number) {
this.collection
.chain()
.find({
$or: [
{ lastContact: { $lt: threshold } },
{ lastContact: null },
]
})
.remove()
}
}
\ No newline at end of file
......@@ -9,11 +9,13 @@ export class DBPeer {
hash: string
first_down: number | null
last_try: number | null
lastContact: number = Math.floor(Date.now() / 1000)
pubkey: string
block: string
signature: string
endpoints: string[]
raw: string
nonWoT: boolean = true // Security measure: a peer is presumed nonWoT.
static json(peer:DBPeer): JSONDBPeer {
return {
......@@ -25,7 +27,7 @@ export class DBPeer {
pubkey: peer.pubkey,
block: peer.block,
signature: peer.signature,
endpoints: peer.endpoints
endpoints: peer.endpoints,
}
}
......
......@@ -78,6 +78,7 @@ export interface NetworkConfDTO {
dos:any
upnp:boolean
httplogs:boolean
nonWoTPeersLimit: number
}
export interface WS2PConfDTO {
......@@ -159,6 +160,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
public memory: boolean,
public nobma: boolean,
public bmaWithCrawler: boolean,
public nonWoTPeersLimit: number,
public proxiesConf: ProxiesConf|undefined,
public ws2p?: {
privateAccess?: boolean
......@@ -181,7 +183,7 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
) {}
static mock() {
return new ConfDTO("", "", [], [], 0, 3600 * 1000, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, new ProxiesConf(), undefined)
return new ConfDTO("", "", [], [], 0, 3600 * 1000, constants.PROOF_OF_WORK.DEFAULT.CPU, 1, constants.PROOF_OF_WORK.DEFAULT.PREFIX, 0, 0, constants.CONTRACT.DEFAULT.C, constants.CONTRACT.DEFAULT.DT, constants.CONTRACT.DEFAULT.DT_REEVAL, 0, constants.CONTRACT.DEFAULT.UD0, 0, 0, constants.CONTRACT.DEFAULT.STEPMAX, constants.CONTRACT.DEFAULT.SIGPERIOD, 0, constants.CONTRACT.DEFAULT.SIGVALIDITY, constants.CONTRACT.DEFAULT.MSVALIDITY, constants.CONTRACT.DEFAULT.SIGQTY, constants.CONTRACT.DEFAULT.SIGSTOCK, constants.CONTRACT.DEFAULT.X_PERCENT, constants.CONTRACT.DEFAULT.PERCENTROT, constants.CONTRACT.DEFAULT.POWDELAY, constants.CONTRACT.DEFAULT.AVGGENTIME, constants.CONTRACT.DEFAULT.MEDIANTIMEBLOCKS, false, 3000, false, constants.BRANCHES.DEFAULT_WINDOW_SIZE, constants.CONTRACT.DEFAULT.IDTYWINDOW, constants.CONTRACT.DEFAULT.MSWINDOW, constants.CONTRACT.DEFAULT.SIGWINDOW, 0, { pub:'', sec:'' }, null, "", "", 0, "", "", "", "", 0, "", "", null, false, "", true, true, false, 100, new ProxiesConf(), undefined)
}
static defaultConf() {
......@@ -211,7 +213,8 @@ export class ConfDTO implements CurrencyConfDTO, KeypairConfDTO, NetworkConfDTO,
"timeout": 3000,
"isolate": false,
"forksize": constants.BRANCHES.DEFAULT_WINDOW_SIZE,
"switchOnHeadAdvance": CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS
"switchOnHeadAdvance": CommonConstants.SWITCH_ON_BRANCH_AHEAD_BY_X_BLOCKS,
"nonWoTPeersLimit": CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT,
};
}
......
......@@ -16,6 +16,7 @@ import {Server} from "../../../../server"
import {BMAConstants} from "./constants"
import {BMALimitation} from "./limiter"
import {Underscore} from "../../../lib/common-libs/underscore"
import {CommonConstants} from "../../../lib/common-libs/constants"
const os = require('os');
const Q = require('q');
......@@ -361,7 +362,8 @@ async function upnpConf (noupnp:boolean, logger:any) {
remoteport: publicPort,
remotehost: null,
remoteipv4: null,
remoteipv6: null
remoteipv6: null,
nonWoTPeersLimit: CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT
}
logger && logger.info('Checking UPnP features...');
if (noupnp) {
......
......@@ -150,7 +150,7 @@ export const CrawlerDependency = {
logger.info('Fetching peering record at %s:%s...', host, port);
let peering = await Contacter.fetchPeer(host, port);
logger.info('Apply peering ...');
await server.PeeringService.submitP(peering, ERASE_IF_ALREADY_RECORDED, !program.nocautious);
await server.PeeringService.submitP(peering, ERASE_IF_ALREADY_RECORDED, !program.nocautious, true);
logger.info('Applied');
let selfPeer = await server.dal.getPeer(server.PeeringService.pubkey);
if (!selfPeer) {
......
......@@ -15,6 +15,6 @@ import {CrawlerConstants} from "./constants"
import {Server} from "../../../../server"
export const cleanLongDownPeers = async (server:Server, now:number) => {
const first_down_limit = now - CrawlerConstants.PEER_LONG_DOWN * 1000;
await server.dal.peerDAL.removePeersDownBefore(first_down_limit)
const first_down_limit = Math.floor((now - CrawlerConstants.PEER_LONG_DOWN * 1000) / 1000)
await server.dal.peerDAL.deletePeersWhoseLastContactIsAbove(first_down_limit)
}
......@@ -91,6 +91,8 @@ export class RemoteSynchronizer extends AbstractSynchronizer {
async init(): Promise<void> {
const peering = await Contacter.fetchPeer(this.host, this.port, RemoteSynchronizer.contacterOptions)
this.peer = PeerDTO.fromJSONObject(peering)
// We save this peer as a trusted peer for future contact
await this.server.PeeringService.submitP(DBPeer.fromPeerDTO(this.peer), false, false, true)
logger.info("Try with %s %s", this.peer.getURL(), this.peer.pubkey.substr(0, 6))
this.node = await connect(this.peer)
;(this.node as any).pubkey = this.peer.pubkey
......
......@@ -25,6 +25,7 @@ import {DBPeer} from "../lib/db/DBPeer"
import {Underscore} from "../lib/common-libs/underscore"
import {CommonConstants} from "../lib/common-libs/constants"
import {DataErrors} from "../lib/common-libs/errors"
import {cleanLongDownPeers} from "../modules/crawler/lib/garbager"
const util = require('util');
const events = require('events');
......@@ -85,7 +86,7 @@ export class PeeringService {
return !!signaturesMatching;
};
submitP(peering:DBPeer, eraseIfAlreadyRecorded = false, cautious = true): Promise<PeerDTO> {
submitP(peering:DBPeer, eraseIfAlreadyRecorded = false, cautious = true, acceptNonWoT = false): Promise<PeerDTO> {
// Force usage of local currency name, do not accept other currencies documents
peering.currency = this.conf.currency || peering.currency;
let thePeerDTO = PeerDTO.fromJSONObject(peering)
......@@ -99,12 +100,42 @@ export class PeeringService {
const hash = thePeerDTO.getHash()
return this.fifoPromiseHandler.pushFIFOPromise<PeerDTO>(hash, async () => {
try {
// First: let's make a cleanup of old peers
await cleanLongDownPeers(this.server, Date.now())
if (makeCheckings) {
let goodSignature = this.checkPeerSignature(thePeerDTO)
if (!goodSignature) {
throw 'Signature from a peer must match';
}
}
// We accept peer documents up to 100 entries, then only member or specific peers are accepted
let isNonWoT = false
if (!acceptNonWoT) {
// Of course we accept our own key
if (peering.pubkey !== this.conf.pair.pub) {
// As well as prefered/priviledged nodes
const isInPrivileged = this.conf.ws2p
&& this.conf.ws2p.privilegedNodes
&& this.conf.ws2p.privilegedNodes.length
&& this.conf.ws2p.privilegedNodes.indexOf(peering.pubkey) !== -1
const isInPrefered = this.conf.ws2p
&& this.conf.ws2p.preferedNodes
&& this.conf.ws2p.preferedNodes.length
&& this.conf.ws2p.preferedNodes.indexOf(peering.pubkey) !== -1
if (!isInPrefered && !isInPrivileged) {
// We also accept all members
const isMember = await this.dal.isMember(this.conf.pair.pub)
if (!isMember) {
isNonWoT = true
// Then as long as we have some room, we accept peers
const hasEnoughRoom = (await this.dal.peerDAL.countNonWoTPeers()) < this.conf.nonWoTPeersLimit
if (!hasEnoughRoom) {
throw Error(DataErrors[DataErrors.PEER_REJECTED])
}
}
}
}
}
if (thePeer.block == constants.PEER.SPECIAL_BLOCK) {
thePeer.block = constants.PEER.SPECIAL_BLOCK;
thePeer.statusTS = 0;
......@@ -118,8 +149,8 @@ export class PeeringService {
thePeer.statusTS = 0;
thePeer.status = 'UP';
}
const current = await this.dal.getBlockCurrent()
if ((!block && current.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS) || (block && current.number - block.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS)) {
const current = await this.dal.getCurrentBlockOrNull()
if (current && ((!block && current.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS) || (block && current.number - block.number > CommonConstants.MAX_AGE_OF_PEER_IN_BLOCKS))) {
throw Error(DataErrors[DataErrors.TOO_OLD_PEER])
}
}
......@@ -167,6 +198,8 @@ export class PeeringService {
peerEntity.last_try = null;
peerEntity.hash = peerEntityOld.getHash()
peerEntity.raw = peerEntityOld.getRaw();
peerEntity.nonWoT = isNonWoT
peerEntity.lastContact = Math.floor(Date.now() / 1000)
await this.dal.savePeer(peerEntity);
this.logger.info('✔ PEER %s', peering.pubkey.substr(0, 8))
let savedPeer = PeerDTO.fromJSONObject(peerEntity).toDBPeer()
......
......@@ -33,14 +33,14 @@ describe('Peers garbaging', () => {
desc: 'Garbage testing',
logs: false,
onDatabaseExecute: async (server:Server) => {
await server.dal.peerDAL.savePeer({ pubkey: 'A', version: 1, currency: 'c', first_down: null, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'B', version: 1, currency: 'c', first_down: 1484827199999, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'C', version: 1, currency: 'c', first_down: 1484827200000, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'D', version: 1, currency: 'c', first_down: 1484820000000, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'A', version: 1, currency: 'c', lastContact: null, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'B', version: 1, currency: 'c', lastContact: 1484827199, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'C', version: 1, currency: 'c', lastContact: 1484827200, statusTS: 1485000000000, block: '2393-H' } as any);
await server.dal.peerDAL.savePeer({ pubkey: 'D', version: 1, currency: 'c', lastContact: 1484820000, statusTS: 1485000000000, block: '2393-H' } as any);
(await server.dal.peerDAL.listAll()).should.have.length(4);
const now = 1485000000000;
const now = 1485000000000
await cleanLongDownPeers(server, now);
(await server.dal.peerDAL.listAll()).should.have.length(2);
(await server.dal.peerDAL.listAll()).should.have.length(1);
}
}]
}
......
......@@ -50,6 +50,7 @@ import {until} from "./test-until"
import {sync} from "./test-sync"
import {expectAnswer, expectError, expectJSON} from "./http-expect"
import {WebSocketServer} from "../../../app/lib/common-libs/websocket"
import {CommonConstants} from "../../../app/lib/common-libs/constants"
const assert = require('assert');
const rp = require('request-promise');
......@@ -228,7 +229,8 @@ export const NewTestingServer = (conf:any) => {
remoteipv4: host,
currency: conf.currency || CURRENCY_NAME,
httpLogs: true,
forksize: conf.forksize || 3
forksize: conf.forksize || 3,
nonWoTPeersLimit: CommonConstants.DEFAULT_NON_WOT_PEERS_LIMIT,
};
if (conf.sigQty === undefined) {
conf.sigQty = 1;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment