[enh] Network scan: create an HttpClient by thread, when refreshing peers

[enh] Upgrade to HttpClient v4.5.6

duniter4j-client/src/main/filtered-resources/log4j.properties

@@ -27,4 +27,7 @@ log4j.logger.org.apache.http=ERROR
 log4j.logger.org.nuiton.util=WARN
 log4j.logger.org.nuiton.config=WARN
 log4j.logger.org.nuiton.converter=WARN
-log4j.logger.org.nuiton.i18n=ERROR
\ No newline at end of file
+log4j.logger.org.nuiton.i18n=ERROR
+
+# Http client connection debug
+#log4j.logger.org.apache.http.impl.conn=DEBUG
\ No newline at end of file

duniter4j-client/src/main/java/org/duniter/client/actions/NetworkAction.java

@@ -94,8 +94,10 @@ public class NetworkAction extends AbstractAction {
         NetworkService service = ServiceLocator.instance().getNetworkService();
 
         if (!autoRefresh) {
+            Long now = System.currentTimeMillis();
             List<Peer> peers = service.getPeers(mainPeer);
             showPeersTable(peers, false);
+            log.info(I18n.t("duniter4j.client.network.executionTime", -System.currentTimeMillis() - now));
         }
         else {
             service.addPeersChangeListener(mainPeer, peers -> showPeersTable(peers, true));

duniter4j-client/src/main/resources/i18n/duniter4j-client_en_GB.properties

@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Fallback to default Duniter peer\: [%s\:%d]
 duniter4j.client.network.action=Display network peers
 duniter4j.client.network.cesiumPlus=Cs+
 duniter4j.client.network.error.outputFieNotWritable=Output file not writable
+duniter4j.client.network.executionTime=Execution time\: %s ms
 duniter4j.client.network.header=Main block [%1$s] computed at [%2$s] validated by [%3$3.2f%%] of peers
 duniter4j.client.network.loadingPeers=Reading network peers...
 duniter4j.client.network.mirror=Mirror

duniter4j-client/src/main/resources/i18n/duniter4j-client_fr_FR.properties

@@ -3,6 +3,7 @@ duniter4j.client.info.peer.fallback=Noeud Duniter (par défaut) \: [%s\:%d]
 duniter4j.client.network.action=Afficher les noeuds Duniter
 duniter4j.client.network.cesiumPlus=Cs+
 duniter4j.client.network.error.outputFieNotWritable=Fichier de sortie non inscriptible
+duniter4j.client.network.executionTime=Temps d'execution \: %s ms
 duniter4j.client.network.header=Bloc principal [%1$s] calculé à [%2$s] validé par [%3$3.2f%%] des noeuds
 duniter4j.client.network.loadingPeers=Lecture des noeuds du réseau...
 duniter4j.client.network.mirror=Mirroir

duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/BlockchainBlock.java

@@ -41,6 +41,17 @@ import java.math.BigInteger;
 @JsonIgnoreProperties(ignoreUnknown=true)
 public class BlockchainBlock implements Serializable {
 
+    public static final String PROPERTY_NUMBER = "number";
+    public static final String PROPERTY_DIVIDEND = "dividend";
+    public static final String PROPERTY_IDENTITIES = "identities";
+    public static final String PROPERTY_JOINERS = "joiners";
+    public static final String PROPERTY_ACTIVES = "actives";
+    public static final String PROPERTY_LEAVERS = "leavers";
+    public static final String PROPERTY_REVOKED = "revoked";
+    public static final String PROPERTY_EXCLUDED = "excluded";
+    public static final String PROPERTY_MEDIAN_TIME = "medianTime";
+
+
     private static final long serialVersionUID = -5598140972293452669L;
     
     private Integer version;

duniter4j-core-client/src/main/java/org/duniter/core/client/service/local/NetworkServiceImpl.java

@@ -115,8 +115,6 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
 
     @Override
     public List<Peer> getPeers(final Peer mainPeer, Filter filter, Sort sort) {
-        //int availableProcessors = Math.min(32, Runtime.getRuntime().availableProcessors());
-        //ExecutorService pool = new ScheduledThreadPoolExecutor(availableProcessors);
         return getPeers(mainPeer, filter, sort, null);
     }
 
@@ -148,7 +146,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
 
     @Override
     public Comparator<Peer> peerComparator(final Sort sort) {
-        return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), (score1, score2) -> score2.compareTo(score1));
+        return Comparator.comparing(peer -> computePeerStatsScore(peer, sort), Comparator.reverseOrder());
     }
 
     @Override

duniter4j-core-client/src/main/java/org/duniter/core/client/util/http/HttpClients.java

+package org.duniter.core.client.util.http;
+
+import org.apache.http.HttpEntityEnclosingRequest;
+import org.apache.http.HttpRequest;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.HttpRequestRetryHandler;
+import org.apache.http.client.config.CookieSpecs;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.config.SocketConfig;
+import org.apache.http.conn.HttpClientConnectionManager;
+import org.apache.http.conn.HttpHostConnectException;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.protocol.HttpContext;
+import org.duniter.core.client.config.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.net.ssl.SSLException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.NoRouteToHostException;
+import java.net.UnknownHostException;
+
+public abstract class HttpClients {
+
+    private static final Logger log = LoggerFactory.getLogger(HttpClients.class);
+
+    private static ThreadLocal<HttpClientConnectionManager> connectionManagerMapper = new ThreadLocal<HttpClientConnectionManager>() {
+        @Override
+        public HttpClientConnectionManager initialValue() {
+            Configuration config = Configuration.instance();
+
+            return createConnectionManager(
+                    config.getNetworkMaxTotalConnections(),
+                    config.getNetworkMaxConnectionsPerRoute(),
+                    config.getNetworkTimeout());
+        }
+    };
+
+    private static ThreadLocal<HttpClient> httpClientsMapper =  new ThreadLocal<HttpClient>() {
+        @Override
+        public HttpClient initialValue() {
+            HttpClientConnectionManager connectionManager= connectionManagerMapper.get();
+            return createHttpClient(connectionManager, 0);
+        }
+    };
+
+    public static HttpClient getThreadHttpClient(final Integer timeout) {
+        if (timeout <= 0) return getThreadHttpClient();
+
+        final HttpClientConnectionManager connectionManager = connectionManagerMapper.get();
+        return createHttpClient(connectionManager, timeout);
+    }
+
+    public static HttpClient getThreadHttpClient() {
+        return httpClientsMapper.get();
+    }
+
+    /**
+     * Remlove client from the thread
+     */
+    public static void remove() {
+        connectionManagerMapper.remove();
+        httpClientsMapper.remove();
+    }
+
+    protected static HttpClient createHttpClient(HttpClientConnectionManager connectionManager, int timeout) {
+        if (timeout <= 0)  {
+            Configuration config = Configuration.instance();
+            timeout = config.getNetworkTimeout();
+        }
+
+        return org.apache.http.impl.client.HttpClients.custom()
+                .setConnectionManager(connectionManager)
+                .setDefaultRequestConfig(createRequestConfig(timeout))
+                .setRetryHandler(createRetryHandler(timeout))
+                .build();
+    }
+
+
+    protected static PoolingHttpClientConnectionManager createConnectionManager(
+            int maxTotalConnections,
+            int maxConnectionsPerRoute,
+            int timeout) {
+        PoolingHttpClientConnectionManager connectionManager
+                = new PoolingHttpClientConnectionManager();
+        connectionManager.setMaxTotal(maxTotalConnections);
+        connectionManager.setDefaultMaxPerRoute(maxConnectionsPerRoute);
+        connectionManager.setDefaultSocketConfig(SocketConfig.custom()
+                .setSoTimeout(timeout).build());
+        return connectionManager;
+    }
+
+    protected static RequestConfig createRequestConfig(int timeout) {
+        return RequestConfig.custom()
+                .setSocketTimeout(timeout).setConnectTimeout(timeout)
+                .setMaxRedirects(1)
+                .setCookieSpec(CookieSpecs.IGNORE_COOKIES)
+                .build();
+    }
+
+
+    protected static HttpRequestRetryHandler createRetryHandler(int timeout) {
+        final int maxRetryCount = (timeout < 1000) ? 2 : 3;
+        return new HttpRequestRetryHandler() {
+            public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
+
+                boolean retrying = true;
+                if (exception instanceof NoRouteToHostException) {
+                    // Bad DNS name
+                    retrying =false;
+                }
+                else if (exception instanceof InterruptedIOException) {
+                    // Timeout
+                    retrying = false;
+                }
+                else if (exception instanceof UnknownHostException) {
+                    // Unknown host
+                    retrying = false;
+                }
+                else if (exception instanceof SSLException) {
+                    // SSL handshake exception
+                    retrying = false;
+                }
+                else if (exception instanceof HttpHostConnectException) {
+                    // Host connect error
+                    retrying = false;
+                }
+
+                if (retrying && executionCount >= maxRetryCount) {
+                    // Do not retry if over max retry count
+                    return false;
+                }
+
+                if (!retrying) {
+                    if (log.isDebugEnabled()) {
+                        log.debug("Failed request to " + HttpClientContext.adapt(context).getRequest().getRequestLine() + ": " + exception.getMessage());
+                    }
+                    return false;
+                }
+
+                HttpClientContext clientContext = HttpClientContext.adapt(context);
+                HttpRequest request = clientContext.getRequest();
+                boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
+                if (idempotent) {
+                    // Retry if the request is considered idempotent
+                    if (log.isDebugEnabled()) log.debug("Failed (but will retry) request to " + request.getRequestLine() + ": " + exception.getMessage());
+                    return true;
+                }
+                return false;
+            }
+        };
+    }
+
+//
+//    private PoolingHttpClientConnectionManager connectionManager;
+//
+//    private static final ThreadLocal<SimpleCache<Integer, HttpClient>> threadLocalCache = new ThreadLocal<SimpleCache<Integer, HttpClient>>() {
+//        @Override
+//        protected SimpleCache<Integer, HttpClient> initialValue() {
+//            int cacheTimeInMillis = 1000 * 60 * 5; // 5 min cache
+//            SimpleCache<Integer, HttpClient> cache = new SimpleCache<Integer, HttpClient>(cacheTimeInMillis*100) {
+//                @Override
+//                public HttpClient load(Integer timeout) {
+//                    return createHttpClient(timeout);
+//                }
+//            };
+//            cache.registerRemoveListener(item -> {
+//                log.debug("Closing HttpClient...");
+//                closeQuietly(item);
+//            });
+//        }
+//    };
+//
+//
+//    protected static HttpClient createHttpClient(int timeout) {
+//        if (connectionManager == null) {
+//            Configuration config = Configuration.instance();
+//            connectionManager = createConnectionManager(
+//                    config.getNetworkMaxTotalConnections(),
+//                    config.getNetworkMaxConnectionsPerRoute(),
+//                    config.getNetworkTimeout());
+//        }
+//
+//        return org.apache.http.impl.client.HttpClients.custom()
+//                .setConnectionManager(connectionManager)
+//                .setDefaultRequestConfig(requestConfigCache.get(timeout))
+//                .setRetryHandler(createRetryHandler(timeout))
+//                .build();
+//    }
+}

duniter4j-core-client/src/main/resources/log4j.properties

@@ -11,6 +11,9 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %5p (%c:%L) - %m%n
 log4j.logger.org.duniter=DEBUG
 log4j.logger.org.duniter.core.client.service.bma.AbstractNetworkService=WARN
 
+# Http client connection debug
+#log4j.logger.org.apache.http.impl.conn=DEBUG
+
 log4j.appender.file=org.apache.log4j.RollingFileAppender
 log4j.appender.file.file=duniter4j-core-client.log
 log4j.appender.file.MaxFileSize=10MB

pom.xml

@@ -244,7 +244,7 @@
       <dependency>
         <groupId>org.apache.httpcomponents</groupId>
         <artifactId>httpclient</artifactId>
-        <version>4.3.3</version>
+        <version>4.5.6</version>
       </dependency>
       <dependency>
         <groupId>org.apache.httpcomponents</groupId>