Commit bbbf68a0 authored by Benoit Lavenier's avatar Benoit Lavenier

[enh] Move ElasticSearch plugins into external project (cesium-plus-pod)

[fix] Fix network scan: filter unknown API; Avoid error on unknown JSON properties
parent 4c67414f
......@@ -15,10 +15,4 @@ Duniter4j is a Java Toolkit for [Duniter](http://duniter.org).
- `duniter4j-core-client`: [a Java API](./src/site/markdown/Java_API.md) to help Java developers to communicate with a Duniter network.
- `duniter4j-elasticsearch`: [a ElastiSearch node](./src/site/markdown/ES.md) used to store (with full-text capabilities) all blockchain data, and additional user data.
* It comes with an [HTTP API](./src/site/markdown/ES_API.md) to store and retrieve all this data.
* This API is used by [Cesium+](https://www.github.com/duniter/cesium) (a Duniter wallet).
......@@ -21,6 +21,7 @@
# Please fill the missing licenses for dependencies :
#
#
#Tue Nov 21 18:04:39 CET 2017
#Wed Jun 20 16:56:38 CEST 2018
commons-primitives--commons-primitives--1.0=The Apache Software License, Version 2.0
dnl.utils--j-text-utils--0.3.3=GNU General Lesser Public License (LGPL) version 3.0
net.sf.opencsv--opencsv--2.3=GNU General Lesser Public License (LGPL) version 3.0
# Duniter4j Configuration
# --------------------------------------
# Duniter4j Client Configuration file
# --------------------------------------
#
# Duniter node
#
duniter4j.node.host=g1.duniter.org
duniter4j.node.port=10901
#
# Cesium+ node (aka Duniter4j-Elasticsearch)
#
duniter4j.node.elasticsearch.host=g1.data.duniter.fr
duniter4j.node.elasticsearch.port=443
......@@ -3,7 +3,9 @@ duniter4j.version=${project.version}
duniter4j.inceptionYear=${project.inceptionYear}
duniter4j.organizationName=${license.organizationName}
#
# Duniter node
#
duniter4j.node.host=192.168.0.5
duniter4j.node.port=10901
......
......@@ -19,8 +19,8 @@ log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p %c - %m%n
# Duniter4j levels
log4j.logger.org.duniter=INFO
log4j.logger.org.duniter.core=WARN
# Avoid warning on leaf not found (Duniter issue)
log4j.logger.org.duniter.core.client.service.local.NetworkServiceImpl=ERROR
# Avoid warning on leaf not found (Duniter issue ?)
#log4j.logger.org.duniter.core.client.service.local.NetworkServiceImpl=WARN
# Other frameworks levels
log4j.logger.org.apache.http=ERROR
......
......@@ -35,6 +35,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Locale;
import java.util.Set;
......@@ -274,6 +275,17 @@ public class Configuration {
}
public URL getNodeElasticSearchUrl() {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
// Force SSL for 443 port
if (getNodeElasticSearchPort() == 443) {
try {
return new URL(applicationConfig.getOption(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey())
.replaceAll("http://", "https://"));
} catch(MalformedURLException e) {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
}
}
else {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
}
}
}
......@@ -47,6 +47,7 @@ public class NetworkPeers implements Serializable {
public String version;
public String currency;
public String status;
public Long statusTS;
public String block;
public String signature;
public String pubkey;
......@@ -79,6 +80,16 @@ public class NetworkPeers implements Serializable {
this.status = status;
}
@JsonGetter("statusTS")
public Long getStatusTS() {
return statusTS;
}
@JsonSetter("statusTS")
public void setStatusTS(Long statusTS) {
this.statusTS = statusTS;
}
public String getBlock() {
return block;
}
......
......@@ -22,6 +22,8 @@ package org.duniter.core.client.model.bma.jackson;
* #L%
*/
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.duniter.core.client.model.bma.BlockchainBlock;
......@@ -63,7 +65,8 @@ public abstract class JacksonUtils extends SimpleModule {
objectMapper.registerModule(module);
// Adding features
//objectMapper.getFactory().configure(JsonGenerator.Feature., true);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
//objectMapper.getFactory().configure(JsonGenerator.Feature.IGNORE_UNKNOWN, true);
return objectMapper;
}
......
......@@ -149,9 +149,9 @@ public class Peer implements LocalEntity<String>, Serializable {
public Peer build() {
int port = this.port != null ? this.port : 80;
boolean useSsl = this.useSsl != null ? this.useSsl :
(port == 443 || this.api == EndpointApi.BMAS.name());
String api = this.api != null ? this.api : EndpointApi.BASIC_MERKLED_API.name();
boolean useSsl = this.useSsl != null ? this.useSsl :
(port == 443 || EndpointApi.BMAS.name().equals(this.api));
Peer ep = new Peer(api, dns, ipv4, ipv6, port, useSsl);
if (StringUtils.isNotBlank(this.epId)) {
ep.setEpId(this.epId);
......
......@@ -51,6 +51,7 @@ import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.service.bma.BmaTechnicalException;
import org.duniter.core.client.service.exception.*;
import org.duniter.core.exception.BusinessException;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ObjectUtils;
import org.duniter.core.util.StringUtils;
......@@ -349,7 +350,7 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
catch (SocketTimeoutException | ConnectTimeoutException e) {
throw new HttpTimeoutException(I18n.t("duniter4j.client.core.timeout"), e);
}
catch (TechnicalException e) {
catch (TechnicalException | BusinessException e) {
throw e;
}
catch (Throwable e) {
......
......@@ -272,7 +272,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
Filter filterDef = new Filter();
filterDef.filterType = null;
filterDef.filterStatus = Peer.PeerStatus.UP;
filterDef.filterEndpoints = ImmutableList.of(EndpointApi.BASIC_MERKLED_API.name(), EndpointApi.BMAS.name());
filterDef.filterEndpoints = ImmutableList.of(EndpointApi.BASIC_MERKLED_API.name(), EndpointApi.BMAS.name(), EndpointApi.WS2P.name());
filterDef.currency = parameters.getCurrency();
// Default sort
......@@ -451,7 +451,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
List<Peer> result = new ArrayList<>();
// If less than 100 node, get it in ONE call
if (leaves.size() < 100) {
if (leaves.size() <= 2000) {
List<Peer> peers = networkRemoteService.getPeers(peer);
if (CollectionUtils.isNotEmpty(peers)) {
......@@ -496,9 +496,12 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
NetworkPeers.Peer peer = networkRemoteService.getPeerLeaf(requestedPeer, leaf);
addEndpointsAsPeers(peer, result, leaf, filterEndpoints);
} catch(HttpNotFoundException | TechnicalException e) {
} catch(HttpNotFoundException hnfe) {
log.debug("Peer not found for leaf=" + leaf);
// skip
} catch(TechnicalException e) {
log.warn("Error while getting peer leaf=" + leaf, e.getMessage());
// skip
}
}
}
......
This diff is collapsed.
This diff is collapsed.
# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
#-------------------------------------------------------------------------------
# Already used licenses in project :
# - ASL, version 2
# - Apache License 2.0
# - Apache License Version 2.0
# - BSD License
# - CC0 1.0 Universal
# - COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
# - Eclipse Public License 1.0
# - General Public License (GPL) v3
# - Indiana University Extreme! Lab Software License, vesion 1.1.1
# - LGPL, version 2.1
# - Lesser General Public License (LGPL) v 3.0
# - Lesser General Public License (LPGL)
# - Lesser General Public License (LPGL) v 2.1
# - MIT License
# - New BSD License
# - Public Domain, per Creative Commons CC0
# - The Apache Software License, Version 2.0
#-------------------------------------------------------------------------------
# Please fill the missing licenses for dependencies :
#
#
#Tue Jan 05 15:24:57 CET 2016
commons-primitives--commons-primitives--1.0=The Apache Software License, Version 2.0
# ======================== Elasticsearch Configuration =========================
#
# NOTE: Elasticsearch comes with reasonable defaults for most settings.
# Before you set out to tweak and tune the configuration, make sure you
# understand what are you trying to accomplish and the consequences.
#
# The primary way of configuring a node is via this file. This template lists
# the most important settings you may want to configure for a production cluster.
#
# Please see the documentation for further information on configuration options:
# <http://www.elastic.co/guide/en/elasticsearch/reference/current/setup-configuration.html>
#
# ---------------------------------- Cluster -----------------------------------
#
# Use a descriptive name for your cluster:
#
cluster.name: duniter4j-es-g1
#
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#
# node.name: node-1
#
# Add custom attributes to the node:
#
# node.rack: r1
#
# ----------------------------------- Paths ------------------------------------
#
# Path to directory where to store the data (separate multiple locations by comma):
#
# path.data: /path/to/data
#
# Path to log files:
#
# path.logs: /path/to/logs
#
# ----------------------------------- Memory -----------------------------------
#
# Lock the memory on startup:
#
# bootstrap.mlockall: true
#
# Make sure that the `ES_HEAP_SIZE` environment variable is set to about half the memory
# available on the system and that the owner of the process is allowed to use this limit.
#
# Elasticsearch performs poorly when the system is swapping the memory.
#
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
# network.host: 192.168.233.118
#
# Set a custom port for HTTP:
#
# http.port: 9200-9300
http.cors.allow-origin: "/.*/"
http.cors.enabled: true
# Internal transport layer
#
# transport.tcp.port: 9210-9220
#
# For more information, see the documentation at:
# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html>
#
# --------------------------------- Discovery ----------------------------------
#
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#
# discovery.zen.ping.unicast.hosts: ["host1", "host2"]
# discovery.zen.ping.unicast.hosts: ["127.0.0.1", ""]
#
# Prevent the "split brain" by configuring the majority of nodes (total number of nodes / 2 + 1):
#
# discovery.zen.minimum_master_nodes: 3
#
# For more information, see the documentation at:
# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-discovery.html>
#
# ---------------------------------- Gateway -----------------------------------
#
# Block initial recovery after a full cluster restart until N nodes are started:
#
# gateway.recover_after_nodes: 3
#
# For more information, see the documentation at:
# <http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-gateway.html>
#
# ---------------------------------- Various -----------------------------------
#
# Disable starting multiple nodes on a single system:
#
# node.max_local_storage_nodes: 1
#
# Require explicit names when deleting indices:
#
# action.destructive_requires_name: true
#
# Security to isolate plugin classpath - /!\ WARNING: should be DISABLE for Duniter4j
#
security.manager.enabled: false
#
# ---------------------------------- Duniter4j ---------------------------------
#
# Enable duniter4j plugins
#
# duniter.enabled: false
#
# Delete then create all indices at startup - /!\ WARNING: DO NOT set to true in production
#
# duniter.indices.reload: true
#
# Default string analyzer
#
duniter.string.analyzer: french
#
# Enabling blockchain synchronization (default: false)
#
duniter.blockchain.enable: true
#
# Force blockchain full synchronization - /!\ WARNING: all user events will be reset to 'unread'
#
# duniter.blockchain.reload: true
# duniter.blockchain.reload.from: 18900
# duniter.blockchain.reload.to: 19000
#
# Duniter node address
#
duniter.host: g1.duniter.org
duniter.port: 10901
# duniter.useSsl: true
#
# Compute statistics on indices (each hour) ? (default: true)
#
# duniter.stats.enable: false
#
# ---------------------------------- Duniter4j security module -------------------
#
# Keyring, use to sign emitted documents (user events, subscription, etc.).
# If not set, random keys will be generated.
#
# duniter.keyring.salt:
# duniter.keyring.password:
#
# Enable security - will restrict HTTP access to only Duniter4j known indices - /!\ WARNING: should be enable for production use
#
duniter.security.enable: true
#
# ---------------------------------- Duniter4j P2P module -------------------------
#
# Enable P2P synchronize between ES peers ? (default: true)
#
# duniter.p2p.enable: false
#
# Enable P2P synchronisation using websocket ? (default: true)
#
# duniter.p2p.ws.enable: false
#
# Time delay (in seconds) to request last documents to peer (e.g. if peer's clock is late). (default: 3600s = 1h)
#
# duniter.p2p.peerTimeOffset: 3600
#
# Enable discovery on network peers, to automatically synchronize this peers (default: true)
#
# duniter.p2p.discovery.enable: false
#
# Pass a list of hosts to always synchronize (default: <empty>)
#
duniter.p2p.includes.endpoints: [
"ES_USER_API g1.data.duniter.fr 443",
"ES_SUBSCRIPTION_API g1.data.duniter.fr 443"
]
#
# Pass a list of pubkeys to always synchronize (default: <empty>)
#
# duniter.p2p.includes.pubkeys: [""]
#
# ---------------------------------- Duniter4j document moderation ---------------
#
# Filter too old document, if time older that 'maxPastDelta' (in seconds). (default: 7200 =2h)
#
# duniter.document.time.maxPastDelta: 7200
#
# Filter document in the futur, if time greater that 'maxFutureDelta' (in seconds). (default: 600 =10min)
#
# duniter.document.time.maxFutureDelta: 600
#
# Allow admin (define in duniter.keyring) to delete documents ? (default: true)
#
# duniter.document.allowAdminDeletion: true
#
# ---------------------------------- Duniter4j Mail module -----------------------
#
# Enable mail module ?
#
duniter.mail.enable: false
#
# Mail: SMTP server configuration (host and port)
#
# duniter.mail.smtp.host: localhost
# duniter.mail.smtp.port: 25
#
# Mail: SMTP server SSL security
#
# duniter.mail.smtp.ssl: true
# duniter.mail.smtp.starttls: true
#
# Mail: SMTP server authentication
#
# duniter.mail.smtp.username:
# duniter.mail.smtp.password:
#
# Mail: 'from' address
#
# duniter.mail.from: no-reply@domain.com
#
# Mail: admin address
#
# duniter.mail.admin: user@domain.com
#
# Mail: subject prefix
#
# duniter.mail.subject.prefix: '[Cesium+]'
# ---------------------------------- Duniter4j Websocket server ----------------------
#
# Websocket port (default: 9400-9410)
#
duniter.ws.port: 9400-9410
#
# ---------------------------------- Duniter4j Subscription module -------------------
#
# Enable subscription module (Need to enable mail features)
#
duniter.subscription.enable: false
#
# Email subscription: Day of the week to trigger weekly (default: 2 = monday)
#
# duniter.subscription.email.dayOfWeek: 2
#
# Email subscription: Hour in day to trigger daily email subscription (default: 3 AM)
#
# duniter.subscription.email.hourOfDay: 3
#
# Email subscription: URL to a Cesium site, for links in the email content (default: https://g1.duniter.fr)
#
# duniter.subscription.email.cesium.url: 'http://domain.com/cesium'
#
# ---------------------------------- Duniter4j User (profile, message) module -------------------
#
#
# Share link: `og:site_name` (default: 'Cesium')
#
# duniter.user.share.site.name: 'Cesium - Ğ1'
#
# Share link: `og:url` - URL to a Cesium site, for links in the email content (default: https://g1.duniter.fr)
#
# duniter.share.cesium.url: 'https://domain.com/cesium'
#
# Share link: Base URL of the ES cluster, to resolve `og:image` URL (default: none => /!\ Will use relative image path)
#
# duniter.share.base.url: 'https://data.domain.com'
\ No newline at end of file
# you can override this using by setting a system property, for example -Des.logger.level=DEBUG
es.logger.level: INFO
rootLogger: ${es.logger.level}, console, file
logger:
# log action execution errors for easier debugging
action: DEBUG
# deprecation logging, turn to DEBUG to see them
deprecation: INFO, deprecation_log_file
# reduce the logging for aws, too much is logged under the default INFO
com.amazonaws: WARN
# aws will try to do some sketchy JMX stuff, but its not needed.
com.amazonaws.jmx.SdkMBeanRegistrySupport: ERROR
com.amazonaws.metrics.AwsSdkMetrics: ERROR
duniter: INFO
#duniter.p2p: TRACE
security: INFO
cluster.metadata: ERROR
cluster.routing.allocation: ERROR
org.duniter: INFO
org.nuiton.i18n: ERROR
org.nuiton.config: ERROR