- blockstat: add 'issuer' field to index

- Peer: Add BMATOR endpoint API

README.md

@@ -3,21 +3,29 @@ duniter4j
 
 duniter4j is a Java Client API for [Duniter](http://duniter.org).
 
-## Architecture
+## Components
 
-duniter4j has four main components :
+Duniter4j has tree main components :
 
- - duniter4j-core-shared: common classes
+ - a command line tool (duniter4j-client), to execute basic operation on a Duniter currency : transfer, view peers, ... 
+  
+ - an API (duniter4j-core-client), that allow developer to access to a Duniter network.
+
+ - a ElastiSearch node (duniter4j-elasticsearch), to add store & full-text capabilities, on blockchain data, user profiles (Cesium+) and more (private message).
+
+## Command line tool
+
+ - Download the file `duniter4j-client-<version>-full-<platform>.zip` from the [lastest releases page](/releases)
+ 
+ - Unzip the archive
  
- - duniter4j-core-client: a Client API to access to a Duniter network.
+ - The open a terminal and call duniter4j.sh
    
- - duniter4j-es-*: ElasticSearch plugins, to store blockchain, user profiles (Cesium+), registry, market and more.
-
- - duniter4j-es-assembly: a standalone assembly with ElasticSearch and embedded plugins 
 
-## Install as ES plugin
+## ElastiSearch node
 
-### Install Java 
+### Prerequisites
+#### Install Java 
 
  - Install Java JRE 8 or more.
  
@@ -36,19 +44,7 @@ sudo apt-get install openjdk-8-jre
        
     No installation need for Windows (include in binaries) 
 
-### Install ElasticSearch 2.3.3
-
- Download lastest release of ElasticSearch
- 
-### Install ElasticSearch plugins
-
-```bash
-   /bin/plugin install mapper-attachments
-   
-   /bin/plugin install https://github.com/duniter/duniter4j/releases/download/0.3.4/duniter4j-elasticsearch-0.3.4.zip
-```
-
-### Install libsodium 
+#### Install libsodium 
 
 [The Sodium crypto library (libsodium)](https://download.libsodium.org/doc/installation/) is a modern, easy-to-use software library for encryption, decryption, signatures, password hashing and more. 
 
@@ -67,7 +63,7 @@ sudo apt-get install openjdk-8-jre
     sudo make install        
 ```
 
-## Install from standalone bundle 
+### Install bundle 
 
  - Install Java (see on top) 
  
@@ -142,6 +138,18 @@ $ ./elasticsearch
 ```
 
 
+### Install on an existing ElasticSearch node
+
+Make sure you have a ElasticSearch v1.4.5 installed.
+ 
+Then install Duniter4j plugins :
+
+```bash
+   /bin/plugin install mapper-attachments
+   
+   /bin/plugin install https://github.com/duniter/duniter4j/releases/download/0.3.4/duniter4j-elasticsearch-0.3.4.zip
+```
+
 ## Use API (Developer)
 
 When a blockchain currency has been indexed, you can test some fun queries :

doc/CLI.md

+# Command line tools
+
+## Installation
+
+ - Download the file `duniter4j-client-<version>-full-<platform>.zip` from the [lastest releases page](https://www.github.com/duniter/duniter4j/releases)
+ 
+ - Unzip the archive;
+   
+ - The open a terminal and execute the script `duniter4j.sh` (or `duniter4j.bat`) :
+ 
+ ```bash
+ cd duniter4j-client-<version>
+ ./duniter4j.sh --help
+ ```
+ 
+
+## Usage
+
+### Execute a transaction
+
+To send money to a pubkey, execute this command :
+
+```bash
+ > ./duniter4j.sh transaction --auth-scrypt --amount 10 --output 2ny7YAdmzReQxAayyJZsyVYwYhVyax2thKcGknmQy5nQ --comment "Thks for Duniter"
+```   
+ 
+### Help
+
+ ```bash
+ > ./duniter4j.sh --help
+ Usage: <main class> [options] [command] [command options]
+   Options:
+     --config
+       Fichier de configuration
+       Default: duniter-client.config
+     --help
+       Affichage de l'aide
+       Default: false
+     -debug
+       Activer les logs de débuggage
+       Default: false
+   Commands:
+     transaction      Effectuer une transaction
+       Usage: transaction [options]
+         Options:
+           --amount
+             Amount
+           --auth-scrypt
+             Authenticate using Scrypt ?
+             Default: true
+           --broadcast
+             Broadcast document sent to all nodes
+             Default: false
+           --comment
+             TX Comment
+           --output
+             Output pubkey
+           --passwd
+             Password (to generate the keypair)
+           -p, --peer
+             Peer address (use format: 'host:port')
+           --salt
+             Salt (to generate the keypair)
+           --scrypt-params
+             Scrypt parameters (N,r,p)
+           --ssl
+             Using SSL connection to node
+             Default: false
+           --timeout
+             HTTP request timeout, in millisecond
+ 
+     network      Afficher les noeuds Duniter
+       Usage: network [options]
+         Options:
+           --continue
+             Continue scanning? (Will refresh on new peer/block).
+             Default: false
+           --output
+             Output CSV file
+           -p, --peer
+             Peer address (use format: 'host:port')
+           --ssl
+             Using SSL connection to node
+             Default: false
+           --timeout
+             HTTP request timeout, in millisecond
+
+ ```
\ No newline at end of file

duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/EndpointApi.java

@@ -26,6 +26,7 @@ package org.duniter.core.client.model.bma;
 public enum EndpointApi {
     BASIC_MERKLED_API,
     BMAS,
+    BMATOR,
     ES_CORE_API,
     ES_USER_API,
     UNDEFINED

duniter4j-core-client/src/main/java/org/duniter/core/client/model/bma/jackson/EndpointDeserializer.java

@@ -69,23 +69,7 @@ public class EndpointDeserializer extends JsonDeserializer<NetworkPeering.Endpoi
         Matcher mather = bmaPattern.matcher(ept);
         if (mather.matches()) {
             endpoint.api = EndpointApi.BASIC_MERKLED_API;
-
-            for(int i=1; i<=mather.groupCount(); i++) {
-                String word = mather.group(i);
-
-                if (StringUtils.isNotBlank(word)) {
-                    if (InetAddressUtils.isIPv4Address(word)) {
-                        endpoint.ipv4 = word;
-                    } else if (InetAddressUtils.isIPv6Address(word)) {
-                        endpoint.ipv6 = word;
-                    } else if (i == mather.groupCount() && word.matches("\\d+")){
-                        endpoint.port = Integer.parseInt(word);
-                    } else {
-                        endpoint.dns = word;
-                    }
-                }
-            }
-
+            parseDefaultFormatEndPoint(mather, endpoint, 1);
             return endpoint;
         }
 
@@ -93,56 +77,42 @@ public class EndpointDeserializer extends JsonDeserializer<NetworkPeering.Endpoi
         mather = bmasPattern.matcher(ept);
         if (mather.matches()) {
             endpoint.api = EndpointApi.BMAS;
-
-            for(int i=1; i<=mather.groupCount(); i++) {
-                String word = mather.group(i);
-
-                if (StringUtils.isNotBlank(word)) {
-                    if (InetAddressUtils.isIPv4Address(word)) {
-                        endpoint.ipv4 = word;
-                    } else if (InetAddressUtils.isIPv6Address(word)) {
-                        endpoint.ipv6 = word;
-                    } else if (i == mather.groupCount() && word.matches("\\d+")){
-                        endpoint.port = Integer.parseInt(word);
-                    } else {
-                        endpoint.dns = word;
-                    }
-                }
-            }
-
+            parseDefaultFormatEndPoint(mather, endpoint, 1);
             return endpoint;
         }
 
         // Other API
         mather = otherApiPattern.matcher(ept);
         if (mather.matches()) {
+            String api = mather.group(1);
             try {
-                endpoint.api = EndpointApi.valueOf(mather.group(1));
+                endpoint.api = EndpointApi.valueOf(api);
+                parseDefaultFormatEndPoint(mather, endpoint, 2);
+                return endpoint;
             } catch(Exception e) {
-                log.warn("Unable to deserialize endpoint: unknown api [" + mather.group(1) + "]");
-                // not known API: skip
-                return null;
+                // Log unknown API (and continue = will skip this endpoint)
+                log.warn("Unable to deserialize endpoint: unknown api [" + api + "]");
             }
+        }
+
+        return null;
+    }
 
-            for(int i=2; i<=mather.groupCount(); i++) {
-                String word = mather.group(i);
-
-                if (StringUtils.isNotBlank(word)) {
-                    if (InetAddressUtils.isIPv4Address(word)) {
-                        endpoint.ipv4 = word;
-                    } else if (InetAddressUtils.isIPv6Address(word)) {
-                        endpoint.ipv6 = word;
-                    } else if (i == mather.groupCount() && word.matches("\\d+")){
-                        endpoint.port = Integer.parseInt(word);
-                    } else {
-                        endpoint.dns = word;
-                    }
+    public static void parseDefaultFormatEndPoint(Matcher matcher, NetworkPeering.Endpoint endpoint, int startGroup) {
+        for(int i=startGroup; i<=matcher.groupCount(); i++) {
+            String word = matcher.group(i);
+
+            if (StringUtils.isNotBlank(word)) {
+                if (InetAddressUtils.isIPv4Address(word)) {
+                    endpoint.ipv4 = word;
+                } else if (InetAddressUtils.isIPv6Address(word)) {
+                    endpoint.ipv6 = word;
+                } else if (i == matcher.groupCount() && word.matches("\\d+")){
+                    endpoint.port = Integer.parseInt(word);
+                } else {
+                    endpoint.dns = word;
                 }
             }
-
-            return endpoint;
         }
-
-        return null;
     }
 }
\ No newline at end of file

duniter4j-core-client/src/main/java/org/duniter/core/client/service/local/NetworkServiceImpl.java

@@ -297,7 +297,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
 
         Runnable getPeersRunnable = () -> {
             if (threadLock.isLocked()) {
-                log.error("Rejected getPeersRunnable() call. Another refresh is already running...");
+                log.debug("Rejected getPeersRunnable() call. Another refresh is already running...");
                 return;
             }
             synchronized (threadLock) {
@@ -331,7 +331,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
 
         Consumer<NetworkPeers.Peer> refreshPeerConsumer = (bmaPeer) -> {
             if (threadLock.isLocked()) {
-                log.error("Rejected refreshPeerConsumer() call. Another refresh is already running...");
+                log.debug("Rejected refreshPeerConsumer() call. Another refresh is already running...");
                 return;
             }
             synchronized (threadLock) {

duniter4j-es-core/src/main/java/org/duniter/elasticsearch/client/Duniter4jClientImpl.java

@@ -25,9 +25,7 @@ package org.duniter.elasticsearch.client;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Joiner;
-import com.google.common.util.concurrent.ListenableFuture;
 import org.apache.commons.collections4.MapUtils;
-import org.apache.http.client.methods.RequestBuilder;
 import org.duniter.core.client.model.bma.jackson.JacksonUtils;
 import org.duniter.core.client.model.elasticsearch.Record;
 import org.duniter.core.client.model.local.LocalEntity;
@@ -36,13 +34,9 @@ import org.duniter.core.util.CollectionUtils;
 import org.duniter.core.util.ObjectUtils;
 import org.duniter.core.util.Preconditions;
 import org.duniter.core.util.StringUtils;
-import org.duniter.core.util.concurrent.CompletableFutures;
-import org.duniter.elasticsearch.dao.AbstractDao;
 import org.duniter.elasticsearch.dao.handler.StringReaderHandler;
 import org.duniter.elasticsearch.exception.AccessDeniedException;
 import org.duniter.elasticsearch.exception.NotFoundException;
-import org.duniter.elasticsearch.threadpool.CompletableActionFuture;
-import org.duniter.elasticsearch.threadpool.RetryPolicy;
 import org.elasticsearch.ElasticsearchException;
 import org.elasticsearch.action.*;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
@@ -85,7 +79,6 @@ import org.elasticsearch.action.search.*;
 import org.elasticsearch.action.suggest.SuggestRequest;
 import org.elasticsearch.action.suggest.SuggestRequestBuilder;
 import org.elasticsearch.action.suggest.SuggestResponse;
-import org.elasticsearch.action.support.PlainListenableActionFuture;
 import org.elasticsearch.action.termvectors.*;
 import org.elasticsearch.action.update.UpdateRequest;
 import org.elasticsearch.action.update.UpdateRequestBuilder;
@@ -99,11 +92,8 @@ import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.logging.ESLogger;
 import org.elasticsearch.common.logging.Loggers;
-import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException;
-import org.elasticsearch.common.util.concurrent.FutureUtils;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHitField;
@@ -111,10 +101,7 @@ import org.elasticsearch.threadpool.ThreadPool;
 
 import java.io.*;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 
 /**
  * Created by Benoit on 08/04/2015.

duniter4j-es-core/src/main/java/org/duniter/elasticsearch/dao/impl/BlockStatDaoImpl.java

@@ -204,6 +204,12 @@ public class BlockStatDaoImpl extends AbstractDao implements BlockStatDao {
                     .field("type", "long")
                     .endObject()
 
+                    // issuer
+                    .startObject(BlockchainBlockStat.PROPERTY_ISSUER)
+                    .field("type", "string")
+                    .field("index", "not_analyzed")
+                    .endObject()
+
                     // hash
                     .startObject(BlockchainBlockStat.PROPERTY_HASH)
                     .field("type", "string")

duniter4j-es-core/src/main/java/org/duniter/elasticsearch/model/BlockchainBlockStat.java

@@ -33,6 +33,7 @@ public class BlockchainBlockStat implements Serializable {
     public static final String PROPERTY_VERSION = "version";
     public static final String PROPERTY_CURRENCY = "currency";
     public static final String PROPERTY_NUMBER = "number";
+    public static final String PROPERTY_ISSUER = "issuer";
     public static final String PROPERTY_HASH = "hash";
     public static final String PROPERTY_MEDIAN_TIME = "medianTime";
     public static final String PROPERTY_MEMBERS_COUNT = "membersCount";
@@ -47,6 +48,7 @@ public class BlockchainBlockStat implements Serializable {
     private int version;
     private String currency;
     private Integer number;
+    private String issuer;
     private String hash;
     private Long medianTime;
     private Integer membersCount;
@@ -79,6 +81,14 @@ public class BlockchainBlockStat implements Serializable {
         this.currency = currency;
     }
 
+    public String getIssuer() {
+        return issuer;
+    }
+
+    public void setIssuer(String issuer) {
+        this.issuer = issuer;
+    }
+
     public BigInteger getDividend() {
         return dividend;
     }
@@ -158,4 +168,5 @@ public class BlockchainBlockStat implements Serializable {
     public void setUnitbase(Integer unitbase) {
         this.unitbase = unitbase;
     }
+
 }

duniter4j-es-core/src/main/java/org/duniter/elasticsearch/service/BlockchainStatsService.java

@@ -34,7 +34,6 @@ import org.duniter.elasticsearch.dao.BlockStatDao;
 import org.duniter.elasticsearch.model.BlockchainBlockStat;
 import org.duniter.elasticsearch.service.changes.ChangeEvent;
 import org.duniter.elasticsearch.threadpool.ThreadPool;
-import org.elasticsearch.action.bulk.BulkRequestBuilder;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.metrics.CounterMetric;
 import org.elasticsearch.common.unit.TimeValue;
@@ -126,8 +125,9 @@ public class BlockchainStatsService extends AbstractBlockchainListenerService {
         BlockchainBlockStat stat = new BlockchainBlockStat();
 
         stat.setNumber(block.getNumber());
-        stat.setHash(block.getHash());
         stat.setCurrency(block.getCurrency());
+        stat.setHash(block.getHash());
+        stat.setIssuer(block.getIssuer());
         stat.setMedianTime(block.getMedianTime());
         stat.setMembersCount(block.getMembersCount());
         stat.setMonetaryMass(block.getMonetaryMass());