Commit 2e0284f4 authored by Benoit Lavenier's avatar Benoit Lavenier

[enh] Network scan: create an HttpClient by thread, when refreshing peers

[enh] Upgrade to HttpClient v4.5.6
parent ea40fcfb
Pipeline #4347 passed with stage
in 34 seconds
......@@ -27,4 +27,7 @@ log4j.logger.org.apache.http=ERROR
log4j.logger.org.nuiton.util=WARN
log4j.logger.org.nuiton.config=WARN
log4j.logger.org.nuiton.converter=WARN
log4j.logger.org.nuiton.i18n=ERROR
\ No newline at end of file
log4j.logger.org.nuiton.i18n=ERROR
# Http client connection debug
#log4j.logger.org.apache.http.impl.conn=DEBUG
\ No newline at end of file
......@@ -94,8 +94,10 @@ public class NetworkAction extends AbstractAction {
NetworkService service = ServiceLocator.instance().getNetworkService();
if (!autoRefresh) {
Long now = System.currentTimeMillis();
List<Peer> peers = service.getPeers(mainPeer);
showPeersTable(peers, false);
log.info(I18n.t("duniter4j.client.network.executionTime", -System.currentTimeMillis() - now));
}
else {
service.addPeersChangeListener(mainPeer, peers -> showPeersTable(peers, true));
......
......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Fallback to default Duniter peer\: [%s\:%d]
duniter4j.client.network.action=Display network peers
duniter4j.client.network.cesiumPlus=Cs+
duniter4j.client.network.error.outputFieNotWritable=Output file not writable
duniter4j.client.network.executionTime=Execution time\: %s ms
duniter4j.client.network.header=Main block [%1$s] computed at [%2$s] validated by [%3$3.2f%%] of peers
duniter4j.client.network.loadingPeers=Reading network peers...
duniter4j.client.network.mirror=Mirror
......
......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Noeud Duniter (par défaut) \: [%s\:%d]
duniter4j.client.network.action=Afficher les noeuds Duniter
duniter4j.client.network.cesiumPlus=Cs+
duniter4j.client.network.error.outputFieNotWritable=Fichier de sortie non inscriptible
duniter4j.client.network.executionTime=Temps d'execution \: %s ms
duniter4j.client.network.header=Bloc principal [%1$s] calculé à [%2$s] validé par [%3$3.2f%%] des noeuds
duniter4j.client.network.loadingPeers=Lecture des noeuds du réseau...
duniter4j.client.network.mirror=Mirroir
......
......@@ -41,6 +41,17 @@ import java.math.BigInteger;
@JsonIgnoreProperties(ignoreUnknown=true)
public class BlockchainBlock implements Serializable {
public static final String PROPERTY_NUMBER = "number";
public static final String PROPERTY_DIVIDEND = "dividend";
public static final String PROPERTY_IDENTITIES = "identities";
public static final String PROPERTY_JOINERS = "joiners";
public static final String PROPERTY_ACTIVES = "actives";
public static final String PROPERTY_LEAVERS = "leavers";
public static final String PROPERTY_REVOKED = "revoked";
public static final String PROPERTY_EXCLUDED = "excluded";
public static final String PROPERTY_MEDIAN_TIME = "medianTime";
private static final long serialVersionUID = -5598140972293452669L;
private Integer version;
......
......@@ -26,48 +26,35 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
import org.duniter.core.beans.InitializingBean;
import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.config.ConfigurationOption;
import org.duniter.core.client.model.bma.Constants;
import org.duniter.core.client.model.bma.Error;
import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.service.bma.BmaTechnicalException;
import org.duniter.core.client.service.exception.*;
import org.duniter.core.client.util.http.HttpClients;
import org.duniter.core.exception.BusinessException;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ObjectUtils;
import org.duniter.core.util.StringUtils;
import org.duniter.core.util.cache.SimpleCache;
import org.duniter.core.util.websocket.WebsocketClientEndpoint;
import org.nuiton.i18n.I18n;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.*;
import java.net.*;
import java.nio.charset.StandardCharsets;
......@@ -84,15 +71,10 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
public static final String URL_PEER_ALIVE = "/blockchain/parameters";
private PoolingHttpClientConnectionManager connectionManager;
protected ObjectMapper objectMapper;
protected Peer defaultPeer;
private boolean debug;
protected Joiner pathJoiner = Joiner.on('/');
protected SimpleCache<Integer, RequestConfig> requestConfigCache;
protected SimpleCache<Integer, HttpClient> httpClientCache;
protected int defaultTimeout;
protected Map<URI, WebsocketClientEndpoint> wsEndPoints = new HashMap<>();
......@@ -115,30 +97,6 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
*/
protected void initCaches() {
Configuration config = Configuration.instance();
int cacheTimeInMillis = config.getNetworkCacheTimeInMillis();
defaultTimeout = config.getNetworkTimeout() > 0 ?
config.getNetworkTimeout() :
Integer.parseInt(ConfigurationOption.NETWORK_TIMEOUT.getDefaultValue());
requestConfigCache = new SimpleCache<Integer, RequestConfig>(cacheTimeInMillis*100) {
@Override
public RequestConfig load(Integer timeout) {
// Use config default timeout, if 0
if (timeout <= 0) timeout = defaultTimeout;
return createRequestConfig(timeout);
}
};
httpClientCache = new SimpleCache<Integer, HttpClient>(cacheTimeInMillis*100) {
@Override
public HttpClient load(Integer timeout) {
return createHttpClient(timeout);
}
};
httpClientCache.registerRemoveListener(item -> {
log.debug("Closing HttpClient...");
closeQuietly(item);
});
}
public void connect(Peer peer) throws PeerConnectionException {
......@@ -152,7 +110,7 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
HttpGet httpGet = new HttpGet(getPath(peer, URL_PEER_ALIVE));
boolean isPeerAlive;
try {
isPeerAlive = executeRequest(httpClientCache.get(0/*=default timeout*/), httpGet);
isPeerAlive = executeRequest(HttpClients.getThreadHttpClient(0), httpGet);
} catch(TechnicalException e) {
this.defaultPeer = null;
throw new PeerConnectionException(e);
......@@ -170,37 +128,40 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
@Override
public void close() throws IOException {
httpClientCache.clear();
requestConfigCache.clear();
if (wsEndPoints.size() != 0) {
for (WebsocketClientEndpoint clientEndPoint: wsEndPoints.values()) {
clientEndPoint.close();
}
wsEndPoints.clear();
}
connectionManager.close();
HttpClients.getThreadHttpClient()
// httpClientCache.clear();
// requestConfigCache.clear();
//
// if (wsEndPoints.size() != 0) {
// for (WebsocketClientEndpoint clientEndPoint: wsEndPoints.values()) {
// clientEndPoint.close();
// }
// wsEndPoints.clear();
// }
//
// if (connectionManager != null) {
// connectionManager.close();
// }
}
public <T> T executeRequest(HttpUriRequest request, Class<? extends T> resultClass) {
return executeRequest(httpClientCache.get(0), request, resultClass);
return executeRequest(HttpClients.getThreadHttpClient(0), request, resultClass);
}
public <T> T executeRequest(HttpUriRequest request, Class<? extends T> resultClass, Class<?> errorClass) {
//return executeRequest(httpClientCache.get(0), request, resultClass, errorClass);
return executeRequest( createHttpClient(0), request, resultClass, errorClass);
//return executeRequest(HttpClients.getThreadHttpClient(0), request, resultClass, errorClass);
return executeRequest( HttpClients.getThreadHttpClient(0), request, resultClass, errorClass);
}
public <T> T executeRequest(String absolutePath, Class<? extends T> resultClass) {
HttpGet httpGet = new HttpGet(getPath(absolutePath));
return executeRequest(httpClientCache.get(0), httpGet, resultClass);
return executeRequest(HttpClients.getThreadHttpClient(0), httpGet, resultClass);
}
public <T> T executeRequest(Peer peer, String absolutePath, Class<? extends T> resultClass) {
HttpGet httpGet = new HttpGet(peer.getUrl() + absolutePath);
return executeRequest(httpClientCache.get(0), httpGet, resultClass);
return executeRequest(HttpClients.getThreadHttpClient(0), httpGet, resultClass);
}
public String getPath(Peer peer, String... absolutePath) {
......@@ -243,94 +204,96 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
}
}
protected PoolingHttpClientConnectionManager createConnectionManager(
int maxTotalConnections,
int maxConnectionsPerRoute,
int timeout) {
PoolingHttpClientConnectionManager connectionManager
= new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(maxTotalConnections);
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
connectionManager.setDefaultSocketConfig(SocketConfig.custom()
.setSoTimeout(timeout).build());
return connectionManager;
}
protected HttpClient createHttpClient(int timeout) {
if (connectionManager == null) {
Configuration config = Configuration.instance();
connectionManager = createConnectionManager(
config.getNetworkMaxTotalConnections(),
config.getNetworkMaxConnectionsPerRoute(),
config.getNetworkTimeout());
}
return HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfigCache.get(timeout))
.setRetryHandler(createRetryHandler(timeout))
.build();
}
protected HttpRequestRetryHandler createRetryHandler(int timeout) {
if (timeout <= 0) timeout = defaultTimeout;
final int maxRetryCount = (timeout < defaultTimeout) ? 2 : 3;
return new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
boolean retrying = true;
if (exception instanceof NoRouteToHostException) {
// Bad DNS name
retrying =false;
}
else if (exception instanceof InterruptedIOException) {
// Timeout
retrying = false;
}
else if (exception instanceof UnknownHostException) {
// Unknown host
retrying = false;
}
else if (exception instanceof SSLException) {
// SSL handshake exception
retrying = false;
}
else if (exception instanceof HttpHostConnectException) {
// Host connect error
retrying = false;
}
if (retrying && executionCount >= maxRetryCount) {
// Do not retry if over max retry count
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
if (!retrying) {
if (debug) log.debug("Failed request to " + request.getRequestLine() + ": " + exception.getMessage());
return false;
}
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// Retry if the request is considered idempotent
if (debug) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
return true;
}
return false;
}
};
}
protected RequestConfig createRequestConfig(int timeout) {
return RequestConfig.custom()
.setSocketTimeout(timeout).setConnectTimeout(timeout)
.setMaxRedirects(1)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
}
// protected PoolingHttpClientConnectionManager createConnectionManager(
// int maxTotalConnections,
// int maxConnectionsPerRoute,
// int timeout) {
// PoolingHttpClientConnectionManager connectionManager
// = new PoolingHttpClientConnectionManager();
// connectionManager.setMaxTotal(maxTotalConnections);
// connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
// connectionManager.setDefaultSocketConfig(SocketConfig.custom()
// .setSoTimeout(timeout).build());
// return connectionManager;
// }
//
// protected HttpClient createHttpClient(int timeout) {
// if (connectionManager == null) {
// Configuration config = Configuration.instance();
// connectionManager = createConnectionManager(
// config.getNetworkMaxTotalConnections(),
// config.getNetworkMaxConnectionsPerRoute(),
// config.getNetworkTimeout());
// }
//
// return HttpClients.custom()
// .setConnectionManager(connectionManager)
// .setDefaultRequestConfig(requestConfigCache.get(timeout))
// .setRetryHandler(httpRetryHandlerCache.get(timeout))
// .build();
// }
//
// protected HttpRequestRetryHandler createRetryHandler(int timeout) {
// if (timeout <= 0) timeout = defaultTimeout;
// final int maxRetryCount = (timeout < defaultTimeout) ? 2 : 3;
// return new HttpRequestRetryHandler() {
// public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
//
// log.warn("Failed request: " + exception.getMessage());
//
// boolean retrying = true;
// if (exception instanceof NoRouteToHostException) {
// // Bad DNS name
// retrying =false;
// }
// else if (exception instanceof InterruptedIOException) {
// // Timeout
// retrying = false;
// }
// else if (exception instanceof UnknownHostException) {
// // Unknown host
// retrying = false;
// }
// else if (exception instanceof SSLException) {
// // SSL handshake exception
// retrying = false;
// }
// else if (exception instanceof HttpHostConnectException) {
// // Host connect error
// retrying = false;
// }
//
// if (retrying && executionCount >= maxRetryCount) {
// // Do not retry if over max retry count
// return false;
// }
//
//
// HttpClientContext clientContext = HttpClientContext.adapt(context);
// HttpRequest request = clientContext.getRequest();
// if (!retrying) {
// if (debug) log.debug("Failed request to " + request.getRequestLine() + ": " + exception.getMessage());
// return false;
// }
//
// boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
// if (idempotent) {
// // Retry if the request is considered idempotent
// if (debug) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
// return true;
// }
// return false;
// }
// };
// }
//
// protected RequestConfig createRequestConfig(int timeout) {
// return RequestConfig.custom()
// .setSocketTimeout(timeout).setConnectTimeout(timeout)
// .setMaxRedirects(1)
// .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
// .build();
// }
protected <T> T executeRequest(HttpClient httpClient, HttpUriRequest request, Class<? extends T> resultClass) {
return executeRequest(httpClient, request, resultClass, Error.class);
......
......@@ -115,8 +115,6 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
@Override
public List<Peer> getPeers(final Peer mainPeer, Filter filter, Sort sort) {
//int availableProcessors = Math.min(32, Runtime.getRuntime().availableProcessors());
//ExecutorService pool = new ScheduledThreadPoolExecutor(availableProcessors);
return getPeers(mainPeer, filter, sort, null);
}
......@@ -148,7 +146,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
@Override
public Comparator<Peer> peerComparator(final Sort sort) {
return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), (score1, score2) -> score2.compareTo(score1));
return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), Comparator.reverseOrder());
}
@Override
......
package org.duniter.core.client.util.http;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.duniter.core.client.config.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
public abstract class HttpClients {
private static final Logger log = LoggerFactory.getLogger(HttpClients.class);
private static ThreadLocal<HttpClientConnectionManager> connectionManagerMapper = new ThreadLocal<HttpClientConnectionManager>() {
@Override
public HttpClientConnectionManager initialValue() {
Configuration config = Configuration.instance();
return createConnectionManager(
config.getNetworkMaxTotalConnections(),
config.getNetworkMaxConnectionsPerRoute(),
config.getNetworkTimeout());
}
};
private static ThreadLocal<HttpClient> httpClientsMapper = new ThreadLocal<HttpClient>() {
@Override
public HttpClient initialValue() {
HttpClientConnectionManager connectionManager= connectionManagerMapper.get();
return createHttpClient(connectionManager, 0);
}
};
public static HttpClient getThreadHttpClient(final Integer timeout) {
if (timeout <= 0) return getThreadHttpClient();
final HttpClientConnectionManager connectionManager = connectionManagerMapper.get();
return createHttpClient(connectionManager, timeout);
}
public static HttpClient getThreadHttpClient() {
return httpClientsMapper.get();
}
/**
* Remlove client from the thread
*/
public static void remove() {
connectionManagerMapper.remove();
httpClientsMapper.remove();
}
protected static HttpClient createHttpClient(HttpClientConnectionManager connectionManager, int timeout) {
if (timeout <= 0) {
Configuration config = Configuration.instance();
timeout = config.getNetworkTimeout();
}
return org.apache.http.impl.client.HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(createRequestConfig(timeout))
.setRetryHandler(createRetryHandler(timeout))
.build();
}
protected static PoolingHttpClientConnectionManager createConnectionManager(
int maxTotalConnections,
int maxConnectionsPerRoute,
int timeout) {
PoolingHttpClientConnectionManager connectionManager
= new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(maxTotalConnections);
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
connectionManager.setDefaultSocketConfig(SocketConfig.custom()
.setSoTimeout(timeout).build());
return connectionManager;
}
protected static RequestConfig createRequestConfig(int timeout) {
return RequestConfig.custom()
.setSocketTimeout(timeout).setConnectTimeout(timeout)
.setMaxRedirects(1)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
}
protected static HttpRequestRetryHandler createRetryHandler(int timeout) {
final int maxRetryCount = (timeout < 1000) ? 2 : 3;
return new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
boolean retrying = true;
if (exception instanceof NoRouteToHostException) {
// Bad DNS name
retrying =false;
}
else if (exception instanceof InterruptedIOException) {
// Timeout
retrying = false;
}
else if (exception instanceof UnknownHostException) {
// Unknown host
retrying = false;
}
else if (exception instanceof SSLException) {
// SSL handshake exception
retrying = false;
}
else if (exception instanceof HttpHostConnectException) {
// Host connect error
retrying = false;
}
if (retrying && executionCount >= maxRetryCount) {
// Do not retry if over max retry count
return false;
}
if (!retrying) {
if (log.isDebugEnabled()) {
log.debug("Failed request to " + HttpClientContext.adapt(context).getRequest().getRequestLine() + ": " + exception.getMessage());
}
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {