From ef2f556a51110b0964d9f76a317b2b044512a8ca Mon Sep 17 00:00:00 2001
From: blavenie <benoit.lavenier@e-is.pro>
Date: Tue, 19 Sep 2017 18:52:53 +0200
Subject: [PATCH] [es-core] start synchro when exists some peers on the
expected endpoint API
---
.../org/duniter/core/client/dao/PeerDao.java | 3 +++
.../client/dao/mem/MemoryPeerDaoImpl.java | 12 +++++++++
.../bma/jackson/EndpointDeserializer.java | 17 ++++++++----
.../elasticsearch/dao/impl/PeerDaoImpl.java | 26 +++++++++++++++++++
.../service/AbstractService.java | 8 ++++++
.../synchro/AbstractSynchroAction.java | 2 +-
.../elasticsearch/synchro/SynchroService.java | 23 +++++++++++-----
7 files changed, 78 insertions(+), 13 deletions(-)
diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/PeerDao.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/PeerDao.java
index 5fbdfaa5..89dcb535 100644
--- a/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/PeerDao.java
+++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/PeerDao.java
@@ -22,9 +22,11 @@ package org.duniter.core.client.dao;
* #L%
*/
+import org.duniter.core.client.model.bma.EndpointApi;
import org.duniter.core.client.model.local.Peer;
import java.util.List;
+import java.util.Set;
/**
* Created by blavenie on 29/12/15.
@@ -41,4 +43,5 @@ public interface PeerDao extends EntityDao<String, Peer> {
void updatePeersAsDown(String currencyId, long maxUpTime);
+ boolean hasPeersUpWithApi(String currencyId, Set<EndpointApi> api);
}
diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/mem/MemoryPeerDaoImpl.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/mem/MemoryPeerDaoImpl.java
index 2a55c192..bf9e4682 100644
--- a/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/mem/MemoryPeerDaoImpl.java
+++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/dao/mem/MemoryPeerDaoImpl.java
@@ -23,6 +23,7 @@ package org.duniter.core.client.dao.mem;
*/
import org.duniter.core.client.dao.PeerDao;
+import org.duniter.core.client.model.bma.EndpointApi;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.util.Preconditions;
@@ -114,4 +115,15 @@ public class MemoryPeerDaoImpl implements PeerDao {
});
}
+
+ @Override
+ public boolean hasPeersUpWithApi(String currencyId, Set<EndpointApi> api) {
+ return getPeersByCurrencyId(currencyId)
+ .stream()
+ .anyMatch(p ->
+ api.contains(EndpointApi.valueOf(p.getApi())) &&
+ p.getStats() != null &&
+ Peer.PeerStatus.UP.equals(p.getStats().getStatus())
+ );
+ }
}
diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/EndpointDeserializer.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/EndpointDeserializer.java
index a70dcb51..1e42352d 100644
--- a/duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/EndpointDeserializer.java
+++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/EndpointDeserializer.java
@@ -53,12 +53,14 @@ public class EndpointDeserializer extends JsonDeserializer<NetworkPeering.Endpoi
private Pattern bmasPattern;
private Pattern ws2pPattern;
private Pattern otherApiPattern;
+ private boolean debug;
public EndpointDeserializer() {
- bmaPattern = Pattern.compile(BMA_API_REGEXP);
- bmasPattern = Pattern.compile(BMAS_API_REGEXP);
- ws2pPattern = Pattern.compile(WS2P_API_REGEXP);
- otherApiPattern = Pattern.compile(OTHER_API_REGEXP);
+ this.bmaPattern = Pattern.compile(BMA_API_REGEXP);
+ this.bmasPattern = Pattern.compile(BMAS_API_REGEXP);
+ this.ws2pPattern = Pattern.compile(WS2P_API_REGEXP);
+ this.otherApiPattern = Pattern.compile(OTHER_API_REGEXP);
+ this.debug = log.isDebugEnabled();
}
@Override
@@ -103,7 +105,12 @@ public class EndpointDeserializer extends JsonDeserializer<NetworkPeering.Endpoi
return endpoint;
} catch(Exception e) {
// Log unknown API (and continue = will skip this endpoint)
- log.warn("Unable to deserialize endpoint: unknown api [" + api + "]");
+ if (debug) {
+ log.warn("Unable to deserialize endpoint: unknown api [" + api + "]", e); // link the exception
+ }
+ else {
+ log.warn("Unable to deserialize endpoint: unknown api [" + api + "]");
+ }
}
}
diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/dao/impl/PeerDaoImpl.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/dao/impl/PeerDaoImpl.java
index 966cd1e8..47b7efa5 100644
--- a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/dao/impl/PeerDaoImpl.java
+++ b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/dao/impl/PeerDaoImpl.java
@@ -23,8 +23,10 @@ package org.duniter.elasticsearch.dao.impl;
*/
import com.fasterxml.jackson.core.JsonProcessingException;
+import org.duniter.core.client.model.bma.EndpointApi;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.exception.TechnicalException;
+import org.duniter.core.util.CollectionUtils;
import org.duniter.core.util.Preconditions;
import org.duniter.core.util.StringUtils;
import org.duniter.elasticsearch.dao.AbstractDao;
@@ -50,6 +52,7 @@ import org.elasticsearch.search.aggregations.metrics.max.Max;
import java.io.IOException;
import java.util.Date;
import java.util.List;
+import java.util.Set;
/**
* Created by blavenie on 29/12/15.
@@ -286,6 +289,29 @@ public class PeerDaoImpl extends AbstractDao implements PeerDao {
}
+ @Override
+ public boolean hasPeersUpWithApi(String currencyId, Set<EndpointApi> api) {
+ SearchRequestBuilder searchRequest = client.prepareSearch(currencyId)
+ .setFetchSource(false)
+ .setTypes(TYPE)
+ .setSize(0);
+
+ // Query = filter on lastUpTime
+ BoolQueryBuilder query = QueryBuilders.boolQuery();
+
+ if (CollectionUtils.isNotEmpty(api)) {
+ query.minimumNumberShouldMatch(api.size());
+ api.forEach(a -> query.should(QueryBuilders.termQuery(Peer.PROPERTY_API, a.name())));
+ }
+
+ query.must(QueryBuilders.nestedQuery(Peer.PROPERTY_STATS, QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery()
+ .filter(QueryBuilders.termQuery(Peer.PROPERTY_STATS + "." + Peer.Stats.PROPERTY_STATUS, Peer.PeerStatus.UP.name())))));
+
+ searchRequest.setQuery(query);
+ SearchResponse response = searchRequest.execute().actionGet();
+ return response.getHits() != null && response.getHits().getTotalHits() > 0;
+ }
+
@Override
public XContentBuilder createTypeMapping() {
try {
diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java
index 1c6b179a..d6a71198 100644
--- a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java
+++ b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/AbstractService.java
@@ -192,6 +192,14 @@ public abstract class AbstractService implements Bean {
}
if (!cryptoService.verify(recordJson, signature, issuer)) {
+
+ if (recordJson.contains("\"socials\":[]")) {
+ recordJson = recordJson.replaceAll(",\"socials\":\\[\\]", "");
+ if (cryptoService.verify(recordJson, signature, issuer)) {
+ return; // ok
+ }
+ }
+
throw new InvalidSignatureException("Invalid signature of JSON string");
}
diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/AbstractSynchroAction.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/AbstractSynchroAction.java
index 6a6bb264..7a0412fe 100644
--- a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/AbstractSynchroAction.java
+++ b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/AbstractSynchroAction.java
@@ -158,7 +158,7 @@ public abstract class AbstractSynchroAction extends AbstractService implements S
}
try {
if (trace) {
- logger.trace(String.format("%s [WS] Processing new change event...", logPrefix));
+ logger.trace(String.format("%s Processing new change event...", logPrefix));
}
JsonNode source = ChangeEvents.readTree(changeEvent.getSource());
diff --git a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/SynchroService.java b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/SynchroService.java
index 2ab2181e..0f055206 100644
--- a/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/SynchroService.java
+++ b/duniter4j-es-core/src/main/java/org/duniter/elasticsearch/synchro/SynchroService.java
@@ -237,24 +237,33 @@ public class SynchroService extends AbstractService {
if (CollectionUtils.isEmpty(currencyIds)) return false;
for (String currencyId: currencyIds) {
- Long lastUpTime = peerDao.getMaxLastUpTime(currencyId);
- if (lastUpTime != null) return true;
+ boolean hasSome = peerDao.hasPeersUpWithApi(currencyId, peerApiFilters);
+ if (hasSome) return true;
}
return false;
}
protected boolean waitPeersReady() throws InterruptedException{
- int tryCounter = 0;
+ final int sleepTime = 10 * 1000 /*10s*/;
+
+
+
+ int maxWaitingDuration = 5 * 6 * sleepTime; // 5 min
+ int waitingDuration = 0;
while (!isReady() && !hasSomePeers()) {
// Wait 10s
- Thread.sleep(10 * 1000);
- tryCounter++;
- if (tryCounter == 6 /*1 min wait*/) {
- logger.warn("Could not start data synchronisation. No Peer found.");
+ Thread.sleep(sleepTime);
+ waitingDuration += sleepTime;
+ if (waitingDuration >= maxWaitingDuration) {
+ logger.warn(String.format("Could not start data synchronisation. No Peer found (after waiting %s min).", waitingDuration/60/1000));
return false; // stop here
}
}
+
+ // Wait again, to make sure all peers have been saved by NetworkService
+ Thread.sleep(sleepTime*2);
+
return true;
}
--
GitLab