Skip to content

GitLab

D
duniter4j
Commit 2e0284f4 authored by Benoit Lavenier's avatar Benoit Lavenier
Browse files
Options

[enh] Network scan: create an HttpClient by thread, when refreshing peers 

[enh] Upgrade to HttpClient v4.5.6
parent ea40fcfb
Pipeline #4347 passed with stage
    in 34 seconds
    Hide whitespace changes
    Inline Side-by-side
    Showing with 327 additions and 153 deletions
    duniter4j-client/src/main/filtered-resources/log4j.properties
    View file @ 2e0284f4
    ......@@ -27,4 +27,7 @@ log4j.logger.org.apache.http=ERROR
    log4j.logger.org.nuiton.util=WARN
    log4j.logger.org.nuiton.config=WARN
    log4j.logger.org.nuiton.converter=WARN
    log4j.logger.org.nuiton.i18n=ERROR
    \ No newline at end of file
    log4j.logger.org.nuiton.i18n=ERROR
    # Http client connection debug
    #log4j.logger.org.apache.http.impl.conn=DEBUG
    \ No newline at end of file
    duniter4j-client/src/main/java/org/duniter/client/actions/NetworkAction.java
    View file @ 2e0284f4
    ......@@ -94,8 +94,10 @@ public class NetworkAction extends AbstractAction {
    NetworkService service = ServiceLocator.instance().getNetworkService();
    if (!autoRefresh) {
    Long now = System.currentTimeMillis();
    List<Peer> peers = service.getPeers(mainPeer);
    showPeersTable(peers, false);
    log.info(I18n.t("duniter4j.client.network.executionTime", -System.currentTimeMillis() - now));
    }
    else {
    service.addPeersChangeListener(mainPeer, peers -> showPeersTable(peers, true));
    ......
    duniter4j-client/src/main/resources/i18n/duniter4j-client_en_GB.properties
    View file @ 2e0284f4
    ......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Fallback to default Duniter peer\: [%s\:%d]
    duniter4j.client.network.action=Display network peers
    duniter4j.client.network.cesiumPlus=Cs+
    duniter4j.client.network.error.outputFieNotWritable=Output file not writable
    duniter4j.client.network.executionTime=Execution time\: %s ms
    duniter4j.client.network.header=Main block [%1$s] computed at [%2$s] validated by [%3$3.2f%%] of peers
    duniter4j.client.network.loadingPeers=Reading network peers...
    duniter4j.client.network.mirror=Mirror
    ......
    duniter4j-client/src/main/resources/i18n/duniter4j-client_fr_FR.properties
    View file @ 2e0284f4
    ......@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Noeud Duniter (par défaut) \: [%s\:%d]
    duniter4j.client.network.action=Afficher les noeuds Duniter
    duniter4j.client.network.cesiumPlus=Cs+
    duniter4j.client.network.error.outputFieNotWritable=Fichier de sortie non inscriptible
    duniter4j.client.network.executionTime=Temps d'execution \: %s ms
    duniter4j.client.network.header=Bloc principal [%1$s] calculé à [%2$s] validé par [%3$3.2f%%] des noeuds
    duniter4j.client.network.loadingPeers=Lecture des noeuds du réseau...
    duniter4j.client.network.mirror=Mirroir
    ......
    duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/BlockchainBlock.java
    View file @ 2e0284f4
    ......@@ -41,6 +41,17 @@ import java.math.BigInteger;
    @JsonIgnoreProperties(ignoreUnknown=true)
    public class BlockchainBlock implements Serializable {
    public static final String PROPERTY_NUMBER = "number";
    public static final String PROPERTY_DIVIDEND = "dividend";
    public static final String PROPERTY_IDENTITIES = "identities";
    public static final String PROPERTY_JOINERS = "joiners";
    public static final String PROPERTY_ACTIVES = "actives";
    public static final String PROPERTY_LEAVERS = "leavers";
    public static final String PROPERTY_REVOKED = "revoked";
    public static final String PROPERTY_EXCLUDED = "excluded";
    public static final String PROPERTY_MEDIAN_TIME = "medianTime";
    private static final long serialVersionUID = -5598140972293452669L;
    private Integer version;
    ......
    duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java
    View file @ 2e0284f4
    ......@@ -26,48 +26,35 @@ import com.fasterxml.jackson.databind.JsonNode;
    import com.fasterxml.jackson.databind.ObjectMapper;
    import com.google.common.base.Charsets;
    import com.google.common.base.Joiner;
    import org.apache.http.HttpEntityEnclosingRequest;
    import org.apache.http.HttpRequest;
    import org.apache.http.HttpResponse;
    import org.apache.http.HttpStatus;
    import org.apache.http.client.HttpClient;
    import org.apache.http.client.HttpRequestRetryHandler;
    import org.apache.http.client.config.CookieSpecs;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.methods.CloseableHttpResponse;
    import org.apache.http.client.methods.HttpGet;
    import org.apache.http.client.methods.HttpUriRequest;
    import org.apache.http.client.protocol.HttpClientContext;
    import org.apache.http.client.utils.URIBuilder;
    import org.apache.http.config.SocketConfig;
    import org.apache.http.conn.ConnectTimeoutException;
    import org.apache.http.conn.HttpHostConnectException;
    import org.apache.http.entity.ContentType;
    import org.apache.http.impl.client.CloseableHttpClient;
    import org.apache.http.impl.client.HttpClients;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.protocol.HttpContext;
    import org.apache.http.util.EntityUtils;
    import org.duniter.core.beans.InitializingBean;
    import org.duniter.core.client.config.Configuration;
    import org.duniter.core.client.config.ConfigurationOption;
    import org.duniter.core.client.model.bma.Constants;
    import org.duniter.core.client.model.bma.Error;
    import org.duniter.core.client.model.bma.jackson.JacksonUtils;
    import org.duniter.core.client.model.local.Peer;
    import org.duniter.core.client.service.bma.BmaTechnicalException;
    import org.duniter.core.client.service.exception.*;
    import org.duniter.core.client.util.http.HttpClients;
    import org.duniter.core.exception.BusinessException;
    import org.duniter.core.exception.TechnicalException;
    import org.duniter.core.util.ObjectUtils;
    import org.duniter.core.util.StringUtils;
    import org.duniter.core.util.cache.SimpleCache;
    import org.duniter.core.util.websocket.WebsocketClientEndpoint;
    import org.nuiton.i18n.I18n;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import javax.net.ssl.SSLException;
    import java.io.*;
    import java.net.*;
    import java.nio.charset.StandardCharsets;
    ......@@ -84,15 +71,10 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
    public static final String URL_PEER_ALIVE = "/blockchain/parameters";
    private PoolingHttpClientConnectionManager connectionManager;
    protected ObjectMapper objectMapper;
    protected Peer defaultPeer;
    private boolean debug;
    protected Joiner pathJoiner = Joiner.on('/');
    protected SimpleCache<Integer, RequestConfig> requestConfigCache;
    protected SimpleCache<Integer, HttpClient> httpClientCache;
    protected int defaultTimeout;
    protected Map<URI, WebsocketClientEndpoint> wsEndPoints = new HashMap<>();
    ......@@ -115,30 +97,6 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
    */
    protected void initCaches() {
    Configuration config = Configuration.instance();
    int cacheTimeInMillis = config.getNetworkCacheTimeInMillis();
    defaultTimeout = config.getNetworkTimeout() > 0 ?
    config.getNetworkTimeout() :
    Integer.parseInt(ConfigurationOption.NETWORK_TIMEOUT.getDefaultValue());
    requestConfigCache = new SimpleCache<Integer, RequestConfig>(cacheTimeInMillis*100) {
    @Override
    public RequestConfig load(Integer timeout) {
    // Use config default timeout, if 0
    if (timeout <= 0) timeout = defaultTimeout;
    return createRequestConfig(timeout);
    }
    };
    httpClientCache = new SimpleCache<Integer, HttpClient>(cacheTimeInMillis*100) {
    @Override
    public HttpClient load(Integer timeout) {
    return createHttpClient(timeout);
    }
    };
    httpClientCache.registerRemoveListener(item -> {
    log.debug("Closing HttpClient...");
    closeQuietly(item);
    });
    }
    public void connect(Peer peer) throws PeerConnectionException {
    ......@@ -152,7 +110,7 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
    HttpGet httpGet = new HttpGet(getPath(peer, URL_PEER_ALIVE));
    boolean isPeerAlive;
    try {
    isPeerAlive = executeRequest(httpClientCache.get(0/*=default timeout*/), httpGet);
    isPeerAlive = executeRequest(HttpClients.getThreadHttpClient(0), httpGet);
    } catch(TechnicalException e) {
    this.defaultPeer = null;
    throw new PeerConnectionException(e);
    ......@@ -170,37 +128,40 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
    @Override
    public void close() throws IOException {
    httpClientCache.clear();
    requestConfigCache.clear();
    if (wsEndPoints.size() != 0) {
    for (WebsocketClientEndpoint clientEndPoint: wsEndPoints.values()) {
    clientEndPoint.close();
    }
    wsEndPoints.clear();
    }
    connectionManager.close();
    HttpClients.getThreadHttpClient()
    // httpClientCache.clear();
    // requestConfigCache.clear();
    //
    // if (wsEndPoints.size() != 0) {
    // for (WebsocketClientEndpoint clientEndPoint: wsEndPoints.values()) {
    // clientEndPoint.close();
    // }
    // wsEndPoints.clear();
    // }
    //
    // if (connectionManager != null) {
    // connectionManager.close();
    // }
    }
    public <T> T executeRequest(HttpUriRequest request, Class<? extends T> resultClass) {
    return executeRequest(httpClientCache.get(0), request, resultClass);
    return executeRequest(HttpClients.getThreadHttpClient(0), request, resultClass);
    }
    public <T> T executeRequest(HttpUriRequest request, Class<? extends T> resultClass, Class<?> errorClass) {
    //return executeRequest(httpClientCache.get(0), request, resultClass, errorClass);
    return executeRequest( createHttpClient(0), request, resultClass, errorClass);
    //return executeRequest(HttpClients.getThreadHttpClient(0), request, resultClass, errorClass);
    return executeRequest( HttpClients.getThreadHttpClient(0), request, resultClass, errorClass);
    }
    public <T> T executeRequest(String absolutePath, Class<? extends T> resultClass) {
    HttpGet httpGet = new HttpGet(getPath(absolutePath));
    return executeRequest(httpClientCache.get(0), httpGet, resultClass);
    return executeRequest(HttpClients.getThreadHttpClient(0), httpGet, resultClass);
    }
    public <T> T executeRequest(Peer peer, String absolutePath, Class<? extends T> resultClass) {
    HttpGet httpGet = new HttpGet(peer.getUrl() + absolutePath);
    return executeRequest(httpClientCache.get(0), httpGet, resultClass);
    return executeRequest(HttpClients.getThreadHttpClient(0), httpGet, resultClass);
    }
    public String getPath(Peer peer, String... absolutePath) {
    ......@@ -243,94 +204,96 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
    }
    }
    protected PoolingHttpClientConnectionManager createConnectionManager(
    int maxTotalConnections,
    int maxConnectionsPerRoute,
    int timeout) {
    PoolingHttpClientConnectionManager connectionManager
    = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(maxTotalConnections);
    connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
    connectionManager.setDefaultSocketConfig(SocketConfig.custom()
    .setSoTimeout(timeout).build());
    return connectionManager;
    }
    protected HttpClient createHttpClient(int timeout) {
    if (connectionManager == null) {
    Configuration config = Configuration.instance();
    connectionManager = createConnectionManager(
    config.getNetworkMaxTotalConnections(),
    config.getNetworkMaxConnectionsPerRoute(),
    config.getNetworkTimeout());
    }
    return HttpClients.custom()
    .setConnectionManager(connectionManager)
    .setDefaultRequestConfig(requestConfigCache.get(timeout))
    .setRetryHandler(createRetryHandler(timeout))
    .build();
    }
    protected HttpRequestRetryHandler createRetryHandler(int timeout) {
    if (timeout <= 0) timeout = defaultTimeout;
    final int maxRetryCount = (timeout < defaultTimeout) ? 2 : 3;
    return new HttpRequestRetryHandler() {
    public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
    boolean retrying = true;
    if (exception instanceof NoRouteToHostException) {
    // Bad DNS name
    retrying =false;
    }
    else if (exception instanceof InterruptedIOException) {
    // Timeout
    retrying = false;
    }
    else if (exception instanceof UnknownHostException) {
    // Unknown host
    retrying = false;
    }
    else if (exception instanceof SSLException) {
    // SSL handshake exception
    retrying = false;
    }
    else if (exception instanceof HttpHostConnectException) {
    // Host connect error
    retrying = false;
    }
    if (retrying && executionCount >= maxRetryCount) {
    // Do not retry if over max retry count
    return false;
    }
    HttpClientContext clientContext = HttpClientContext.adapt(context);
    HttpRequest request = clientContext.getRequest();
    if (!retrying) {
    if (debug) log.debug("Failed request to " + request.getRequestLine() + ": " + exception.getMessage());
    return false;
    }
    boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
    if (idempotent) {
    // Retry if the request is considered idempotent
    if (debug) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
    return true;
    }
    return false;
    }
    };
    }
    protected RequestConfig createRequestConfig(int timeout) {
    return RequestConfig.custom()
    .setSocketTimeout(timeout).setConnectTimeout(timeout)
    .setMaxRedirects(1)
    .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
    .build();
    }
    // protected PoolingHttpClientConnectionManager createConnectionManager(
    // int maxTotalConnections,
    // int maxConnectionsPerRoute,
    // int timeout) {
    // PoolingHttpClientConnectionManager connectionManager
    // = new PoolingHttpClientConnectionManager();
    // connectionManager.setMaxTotal(maxTotalConnections);
    // connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
    // connectionManager.setDefaultSocketConfig(SocketConfig.custom()
    // .setSoTimeout(timeout).build());
    // return connectionManager;
    // }
    //
    // protected HttpClient createHttpClient(int timeout) {
    // if (connectionManager == null) {
    // Configuration config = Configuration.instance();
    // connectionManager = createConnectionManager(
    // config.getNetworkMaxTotalConnections(),
    // config.getNetworkMaxConnectionsPerRoute(),
    // config.getNetworkTimeout());
    // }
    //
    // return HttpClients.custom()
    // .setConnectionManager(connectionManager)
    // .setDefaultRequestConfig(requestConfigCache.get(timeout))
    // .setRetryHandler(httpRetryHandlerCache.get(timeout))
    // .build();
    // }
    //
    // protected HttpRequestRetryHandler createRetryHandler(int timeout) {
    // if (timeout <= 0) timeout = defaultTimeout;
    // final int maxRetryCount = (timeout < defaultTimeout) ? 2 : 3;
    // return new HttpRequestRetryHandler() {
    // public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
    //
    // log.warn("Failed request: " + exception.getMessage());
    //
    // boolean retrying = true;
    // if (exception instanceof NoRouteToHostException) {
    // // Bad DNS name
    // retrying =false;
    // }
    // else if (exception instanceof InterruptedIOException) {
    // // Timeout
    // retrying = false;
    // }
    // else if (exception instanceof UnknownHostException) {
    // // Unknown host
    // retrying = false;
    // }
    // else if (exception instanceof SSLException) {
    // // SSL handshake exception
    // retrying = false;
    // }
    // else if (exception instanceof HttpHostConnectException) {
    // // Host connect error
    // retrying = false;
    // }
    //
    // if (retrying && executionCount >= maxRetryCount) {
    // // Do not retry if over max retry count
    // return false;
    // }
    //
    //
    // HttpClientContext clientContext = HttpClientContext.adapt(context);
    // HttpRequest request = clientContext.getRequest();
    // if (!retrying) {
    // if (debug) log.debug("Failed request to " + request.getRequestLine() + ": " + exception.getMessage());
    // return false;
    // }
    //
    // boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
    // if (idempotent) {
    // // Retry if the request is considered idempotent
    // if (debug) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
    // return true;
    // }
    // return false;
    // }
    // };
    // }
    //
    // protected RequestConfig createRequestConfig(int timeout) {
    // return RequestConfig.custom()
    // .setSocketTimeout(timeout).setConnectTimeout(timeout)
    // .setMaxRedirects(1)
    // .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
    // .build();
    // }
    protected <T> T executeRequest(HttpClient httpClient, HttpUriRequest request, Class<? extends T> resultClass) {
    return executeRequest(httpClient, request, resultClass, Error.class);
    ......
    duniter4j-core-client/src/main/java/org/duniter/core/client/service/local/NetworkServiceImpl.java
    View file @ 2e0284f4
    ......@@ -115,8 +115,6 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
    @Override
    public List<Peer> getPeers(final Peer mainPeer, Filter filter, Sort sort) {
    //int availableProcessors = Math.min(32, Runtime.getRuntime().availableProcessors());
    //ExecutorService pool = new ScheduledThreadPoolExecutor(availableProcessors);
    return getPeers(mainPeer, filter, sort, null);
    }
    ......@@ -148,7 +146,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
    @Override
    public Comparator<Peer> peerComparator(final Sort sort) {
    return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), (score1, score2) -> score2.compareTo(score1));
    return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), Comparator.reverseOrder());
    }
    @Override
    ......
    duniter4j-core-client/src/main/java/org/duniter/core/client/util/http/HttpClients.java 0 → 100644
    View file @ 2e0284f4
    package org.duniter.core.client.util.http;
    import org.apache.http.HttpEntityEnclosingRequest;
    import org.apache.http.HttpRequest;
    import org.apache.http.client.HttpClient;
    import org.apache.http.client.HttpRequestRetryHandler;
    import org.apache.http.client.config.CookieSpecs;
    import org.apache.http.client.config.RequestConfig;
    import org.apache.http.client.protocol.HttpClientContext;
    import org.apache.http.config.SocketConfig;
    import org.apache.http.conn.HttpClientConnectionManager;
    import org.apache.http.conn.HttpHostConnectException;
    import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
    import org.apache.http.protocol.HttpContext;
    import org.duniter.core.client.config.Configuration;
    import org.slf4j.Logger;
    import org.slf4j.LoggerFactory;
    import javax.net.ssl.SSLException;
    import java.io.IOException;
    import java.io.InterruptedIOException;
    import java.net.NoRouteToHostException;
    import java.net.UnknownHostException;
    public abstract class HttpClients {
    private static final Logger log = LoggerFactory.getLogger(HttpClients.class);
    private static ThreadLocal<HttpClientConnectionManager> connectionManagerMapper = new ThreadLocal<HttpClientConnectionManager>() {
    @Override
    public HttpClientConnectionManager initialValue() {
    Configuration config = Configuration.instance();
    return createConnectionManager(
    config.getNetworkMaxTotalConnections(),
    config.getNetworkMaxConnectionsPerRoute(),
    config.getNetworkTimeout());
    }
    };
    private static ThreadLocal<HttpClient> httpClientsMapper = new ThreadLocal<HttpClient>() {
    @Override
    public HttpClient initialValue() {
    HttpClientConnectionManager connectionManager= connectionManagerMapper.get();
    return createHttpClient(connectionManager, 0);
    }
    };
    public static HttpClient getThreadHttpClient(final Integer timeout) {
    if (timeout <= 0) return getThreadHttpClient();
    final HttpClientConnectionManager connectionManager = connectionManagerMapper.get();
    return createHttpClient(connectionManager, timeout);
    }
    public static HttpClient getThreadHttpClient() {
    return httpClientsMapper.get();
    }
    /**
    * Remlove client from the thread
    */
    public static void remove() {
    connectionManagerMapper.remove();
    httpClientsMapper.remove();
    }
    protected static HttpClient createHttpClient(HttpClientConnectionManager connectionManager, int timeout) {
    if (timeout <= 0) {
    Configuration config = Configuration.instance();
    timeout = config.getNetworkTimeout();
    }
    return org.apache.http.impl.client.HttpClients.custom()
    .setConnectionManager(connectionManager)
    .setDefaultRequestConfig(createRequestConfig(timeout))
    .setRetryHandler(createRetryHandler(timeout))
    .build();
    }
    protected static PoolingHttpClientConnectionManager createConnectionManager(
    int maxTotalConnections,
    int maxConnectionsPerRoute,
    int timeout) {
    PoolingHttpClientConnectionManager connectionManager
    = new PoolingHttpClientConnectionManager();
    connectionManager.setMaxTotal(maxTotalConnections);
    connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
    connectionManager.setDefaultSocketConfig(SocketConfig.custom()
    .setSoTimeout(timeout).build());
    return connectionManager;
    }
    protected static RequestConfig createRequestConfig(int timeout) {
    return RequestConfig.custom()
    .setSocketTimeout(timeout).setConnectTimeout(timeout)
    .setMaxRedirects(1)
    .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
    .build();
    }
    protected static HttpRequestRetryHandler createRetryHandler(int timeout) {
    final int maxRetryCount = (timeout < 1000) ? 2 : 3;
    return new HttpRequestRetryHandler() {
    public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
    boolean retrying = true;
    if (exception instanceof NoRouteToHostException) {
    // Bad DNS name
    retrying =false;
    }
    else if (exception instanceof InterruptedIOException) {
    // Timeout
    retrying = false;
    }
    else if (exception instanceof UnknownHostException) {
    // Unknown host
    retrying = false;
    }
    else if (exception instanceof SSLException) {
    // SSL handshake exception
    retrying = false;
    }
    else if (exception instanceof HttpHostConnectException) {
    // Host connect error
    retrying = false;
    }
    if (retrying && executionCount >= maxRetryCount) {
    // Do not retry if over max retry count
    return false;
    }
    if (!retrying) {
    if (log.isDebugEnabled()) {
    log.debug("Failed request to " + HttpClientContext.adapt(context).getRequest().getRequestLine() + ": " + exception.getMessage());
    }
    return false;
    }
    HttpClientContext clientContext = HttpClientContext.adapt(context);
    HttpRequest request = clientContext.getRequest();
    boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
    if (idempotent) {