Commit 2e0284f4 authored by Benoit Lavenier's avatar Benoit Lavenier

[enh] Network scan: create an HttpClient by thread, when refreshing peers

[enh] Upgrade to HttpClient v4.5.6
parent ea40fcfb
Pipeline #4347 passed with stage
in 34 seconds
......@@ -27,4 +27,7 @@ log4j.logger.org.apache.http=ERROR
log4j.logger.org.nuiton.util=WARN
log4j.logger.org.nuiton.config=WARN
log4j.logger.org.nuiton.converter=WARN
log4j.logger.org.nuiton.i18n=ERROR
\ No newline at end of file
log4j.logger.org.nuiton.i18n=ERROR
# Http client connection debug
#log4j.logger.org.apache.http.impl.conn=DEBUG
\ No newline at end of file
......@@ -94,8 +94,10 @@ public class NetworkAction extends AbstractAction {
NetworkService service = ServiceLocator.instance().getNetworkService();
if (!autoRefresh) {
Long now = System.currentTimeMillis();
List<Peer> peers = service.getPeers(mainPeer);
showPeersTable(peers, false);
log.info(I18n.t("duniter4j.client.network.executionTime", -System.currentTimeMillis() - now));
}
else {
service.addPeersChangeListener(mainPeer, peers -> showPeersTable(peers, true));
......
......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Fallback to default Duniter peer\: [%s\:%d]
duniter4j.client.network.action=Display network peers
duniter4j.client.network.cesiumPlus=Cs+
duniter4j.client.network.error.outputFieNotWritable=Output file not writable
duniter4j.client.network.executionTime=Execution time\: %s ms
duniter4j.client.network.header=Main block [%1$s] computed at [%2$s] validated by [%3$3.2f%%] of peers
duniter4j.client.network.loadingPeers=Reading network peers...
duniter4j.client.network.mirror=Mirror
......
......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Noeud Duniter (par défaut) \: [%s\:%d]
duniter4j.client.network.action=Afficher les noeuds Duniter
duniter4j.client.network.cesiumPlus=Cs+
duniter4j.client.network.error.outputFieNotWritable=Fichier de sortie non inscriptible
duniter4j.client.network.executionTime=Temps d'execution \: %s ms
duniter4j.client.network.header=Bloc principal [%1$s] calculé à [%2$s] validé par [%3$3.2f%%] des noeuds
duniter4j.client.network.loadingPeers=Lecture des noeuds du réseau...
duniter4j.client.network.mirror=Mirroir
......
......@@ -41,6 +41,17 @@ import java.math.BigInteger;
@JsonIgnoreProperties(ignoreUnknown=true)
public class BlockchainBlock implements Serializable {
public static final String PROPERTY_NUMBER = "number";
public static final String PROPERTY_DIVIDEND = "dividend";
public static final String PROPERTY_IDENTITIES = "identities";
public static final String PROPERTY_JOINERS = "joiners";
public static final String PROPERTY_ACTIVES = "actives";
public static final String PROPERTY_LEAVERS = "leavers";
public static final String PROPERTY_REVOKED = "revoked";
public static final String PROPERTY_EXCLUDED = "excluded";
public static final String PROPERTY_MEDIAN_TIME = "medianTime";
private static final long serialVersionUID = -5598140972293452669L;
private Integer version;
......
......@@ -115,8 +115,6 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
@Override
public List<Peer> getPeers(final Peer mainPeer, Filter filter, Sort sort) {
//int availableProcessors = Math.min(32, Runtime.getRuntime().availableProcessors());
//ExecutorService pool = new ScheduledThreadPoolExecutor(availableProcessors);
return getPeers(mainPeer, filter, sort, null);
}
......@@ -148,7 +146,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
@Override
public Comparator<Peer> peerComparator(final Sort sort) {
return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), (score1, score2) -> score2.compareTo(score1));
return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), Comparator.reverseOrder());
}
@Override
......
package org.duniter.core.client.util.http;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.client.HttpClient;
import org.apache.http.client.HttpRequestRetryHandler;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.SocketConfig;
import org.apache.http.conn.HttpClientConnectionManager;
import org.apache.http.conn.HttpHostConnectException;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.duniter.core.client.config.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.net.ssl.SSLException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.net.NoRouteToHostException;
import java.net.UnknownHostException;
public abstract class HttpClients {
private static final Logger log = LoggerFactory.getLogger(HttpClients.class);
private static ThreadLocal<HttpClientConnectionManager> connectionManagerMapper = new ThreadLocal<HttpClientConnectionManager>() {
@Override
public HttpClientConnectionManager initialValue() {
Configuration config = Configuration.instance();
return createConnectionManager(
config.getNetworkMaxTotalConnections(),
config.getNetworkMaxConnectionsPerRoute(),
config.getNetworkTimeout());
}
};
private static ThreadLocal<HttpClient> httpClientsMapper = new ThreadLocal<HttpClient>() {
@Override
public HttpClient initialValue() {
HttpClientConnectionManager connectionManager= connectionManagerMapper.get();
return createHttpClient(connectionManager, 0);
}
};
public static HttpClient getThreadHttpClient(final Integer timeout) {
if (timeout <= 0) return getThreadHttpClient();
final HttpClientConnectionManager connectionManager = connectionManagerMapper.get();
return createHttpClient(connectionManager, timeout);
}
public static HttpClient getThreadHttpClient() {
return httpClientsMapper.get();
}
/**
* Remlove client from the thread
*/
public static void remove() {
connectionManagerMapper.remove();
httpClientsMapper.remove();
}
protected static HttpClient createHttpClient(HttpClientConnectionManager connectionManager, int timeout) {
if (timeout <= 0) {
Configuration config = Configuration.instance();
timeout = config.getNetworkTimeout();
}
return org.apache.http.impl.client.HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(createRequestConfig(timeout))
.setRetryHandler(createRetryHandler(timeout))
.build();
}
protected static PoolingHttpClientConnectionManager createConnectionManager(
int maxTotalConnections,
int maxConnectionsPerRoute,
int timeout) {
PoolingHttpClientConnectionManager connectionManager
= new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(maxTotalConnections);
connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
connectionManager.setDefaultSocketConfig(SocketConfig.custom()
.setSoTimeout(timeout).build());
return connectionManager;
}
protected static RequestConfig createRequestConfig(int timeout) {
return RequestConfig.custom()
.setSocketTimeout(timeout).setConnectTimeout(timeout)
.setMaxRedirects(1)
.setCookieSpec(CookieSpecs.IGNORE_COOKIES)
.build();
}
protected static HttpRequestRetryHandler createRetryHandler(int timeout) {
final int maxRetryCount = (timeout < 1000) ? 2 : 3;
return new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
boolean retrying = true;
if (exception instanceof NoRouteToHostException) {
// Bad DNS name
retrying =false;
}
else if (exception instanceof InterruptedIOException) {
// Timeout
retrying = false;
}
else if (exception instanceof UnknownHostException) {
// Unknown host
retrying = false;
}
else if (exception instanceof SSLException) {
// SSL handshake exception
retrying = false;
}
else if (exception instanceof HttpHostConnectException) {
// Host connect error
retrying = false;
}
if (retrying && executionCount >= maxRetryCount) {
// Do not retry if over max retry count
return false;
}
if (!retrying) {
if (log.isDebugEnabled()) {
log.debug("Failed request to " + HttpClientContext.adapt(context).getRequest().getRequestLine() + ": " + exception.getMessage());
}
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
if (idempotent) {
// Retry if the request is considered idempotent
if (log.isDebugEnabled()) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
return true;
}
return false;
}
};
}
//
// private PoolingHttpClientConnectionManager connectionManager;
//
// private static final ThreadLocal<SimpleCache<Integer, HttpClient>> threadLocalCache = new ThreadLocal<SimpleCache<Integer, HttpClient>>() {
// @Override
// protected SimpleCache<Integer, HttpClient> initialValue() {
// int cacheTimeInMillis = 1000 * 60 * 5; // 5 min cache
// SimpleCache<Integer, HttpClient> cache = new SimpleCache<Integer, HttpClient>(cacheTimeInMillis*100) {
// @Override
// public HttpClient load(Integer timeout) {
// return createHttpClient(timeout);
// }
// };
// cache.registerRemoveListener(item -> {
// log.debug("Closing HttpClient...");
// closeQuietly(item);
// });
// }
// };
//
//
// protected static HttpClient createHttpClient(int timeout) {
// if (connectionManager == null) {
// Configuration config = Configuration.instance();
// connectionManager = createConnectionManager(
// config.getNetworkMaxTotalConnections(),
// config.getNetworkMaxConnectionsPerRoute(),
// config.getNetworkTimeout());
// }
//
// return org.apache.http.impl.client.HttpClients.custom()
// .setConnectionManager(connectionManager)
// .setDefaultRequestConfig(requestConfigCache.get(timeout))
// .setRetryHandler(createRetryHandler(timeout))
// .build();
// }
}
......@@ -11,6 +11,9 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %5p (%c:%L) - %m%n
log4j.logger.org.duniter=DEBUG
log4j.logger.org.duniter.core.client.service.bma.AbstractNetworkService=WARN
# Http client connection debug
#log4j.logger.org.apache.http.impl.conn=DEBUG
log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.file=duniter4j-core-client.log
log4j.appender.file.MaxFileSize=10MB
......
......@@ -244,7 +244,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.3.3</version>
<version>4.5.6</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment