Commit bbbf68a0 authored by Benoit Lavenier's avatar Benoit Lavenier
Browse files

[enh] Move ElasticSearch plugins into external project (cesium-plus-pod)

[fix] Fix network scan: filter unknown API; Avoid error on unknown JSON properties
parent 4c67414f
......@@ -15,10 +15,4 @@ Duniter4j is a Java Toolkit for [Duniter](http://duniter.org).
- `duniter4j-core-client`: [a Java API](./src/site/markdown/Java_API.md) to help Java developers to communicate with a Duniter network.
- `duniter4j-elasticsearch`: [a ElastiSearch node](./src/site/markdown/ES.md) used to store (with full-text capabilities) all blockchain data, and additional user data.
* It comes with an [HTTP API](./src/site/markdown/ES_API.md) to store and retrieve all this data.
* This API is used by [Cesium+](https://www.github.com/duniter/cesium) (a Duniter wallet).
......@@ -21,6 +21,7 @@
# Please fill the missing licenses for dependencies :
#
#
#Tue Nov 21 18:04:39 CET 2017
#Wed Jun 20 16:56:38 CEST 2018
commons-primitives--commons-primitives--1.0=The Apache Software License, Version 2.0
dnl.utils--j-text-utils--0.3.3=GNU General Lesser Public License (LGPL) version 3.0
net.sf.opencsv--opencsv--2.3=GNU General Lesser Public License (LGPL) version 3.0
# Duniter4j Configuration
# --------------------------------------
# Duniter4j Client Configuration file
# --------------------------------------
#
# Duniter node
#
duniter4j.node.host=g1.duniter.org
duniter4j.node.port=10901
#
# Cesium+ node (aka Duniter4j-Elasticsearch)
#
duniter4j.node.elasticsearch.host=g1.data.duniter.fr
duniter4j.node.elasticsearch.port=443
......@@ -3,7 +3,9 @@ duniter4j.version=${project.version}
duniter4j.inceptionYear=${project.inceptionYear}
duniter4j.organizationName=${license.organizationName}
#
# Duniter node
#
duniter4j.node.host=192.168.0.5
duniter4j.node.port=10901
......
......@@ -19,8 +19,8 @@ log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p %c - %m%n
# Duniter4j levels
log4j.logger.org.duniter=INFO
log4j.logger.org.duniter.core=WARN
# Avoid warning on leaf not found (Duniter issue)
log4j.logger.org.duniter.core.client.service.local.NetworkServiceImpl=ERROR
# Avoid warning on leaf not found (Duniter issue ?)
#log4j.logger.org.duniter.core.client.service.local.NetworkServiceImpl=WARN
# Other frameworks levels
log4j.logger.org.apache.http=ERROR
......
......@@ -35,6 +35,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Locale;
import java.util.Set;
......@@ -274,6 +275,17 @@ public class Configuration {
}
public URL getNodeElasticSearchUrl() {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
// Force SSL for 443 port
if (getNodeElasticSearchPort() == 443) {
try {
return new URL(applicationConfig.getOption(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey())
.replaceAll("http://", "https://"));
} catch(MalformedURLException e) {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
}
}
else {
return applicationConfig.getOptionAsURL(ConfigurationOption.NODE_ELASTICSEARCH_URL.getKey());
}
}
}
......@@ -47,6 +47,7 @@ public class NetworkPeers implements Serializable {
public String version;
public String currency;
public String status;
public Long statusTS;
public String block;
public String signature;
public String pubkey;
......@@ -79,6 +80,16 @@ public class NetworkPeers implements Serializable {
this.status = status;
}
@JsonGetter("statusTS")
public Long getStatusTS() {
return statusTS;
}
@JsonSetter("statusTS")
public void setStatusTS(Long statusTS) {
this.statusTS = statusTS;
}
public String getBlock() {
return block;
}
......
......@@ -22,6 +22,8 @@ package org.duniter.core.client.model.bma.jackson;
* #L%
*/
import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.module.SimpleModule;
import org.duniter.core.client.model.bma.BlockchainBlock;
......@@ -63,7 +65,8 @@ public abstract class JacksonUtils extends SimpleModule {
objectMapper.registerModule(module);
// Adding features
//objectMapper.getFactory().configure(JsonGenerator.Feature., true);
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
//objectMapper.getFactory().configure(JsonGenerator.Feature.IGNORE_UNKNOWN, true);
return objectMapper;
}
......
......@@ -149,9 +149,9 @@ public class Peer implements LocalEntity<String>, Serializable {
public Peer build() {
int port = this.port != null ? this.port : 80;
boolean useSsl = this.useSsl != null ? this.useSsl :
(port == 443 || this.api == EndpointApi.BMAS.name());
String api = this.api != null ? this.api : EndpointApi.BASIC_MERKLED_API.name();
boolean useSsl = this.useSsl != null ? this.useSsl :
(port == 443 || EndpointApi.BMAS.name().equals(this.api));
Peer ep = new Peer(api, dns, ipv4, ipv6, port, useSsl);
if (StringUtils.isNotBlank(this.epId)) {
ep.setEpId(this.epId);
......
......@@ -51,6 +51,7 @@ import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.service.bma.BmaTechnicalException;
import org.duniter.core.client.service.exception.*;
import org.duniter.core.exception.BusinessException;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ObjectUtils;
import org.duniter.core.util.StringUtils;
......@@ -349,7 +350,7 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean
catch (SocketTimeoutException | ConnectTimeoutException e) {
throw new HttpTimeoutException(I18n.t("duniter4j.client.core.timeout"), e);
}
catch (TechnicalException e) {
catch (TechnicalException | BusinessException e) {
throw e;
}
catch (Throwable e) {
......
......@@ -272,7 +272,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
Filter filterDef = new Filter();
filterDef.filterType = null;
filterDef.filterStatus = Peer.PeerStatus.UP;
filterDef.filterEndpoints = ImmutableList.of(EndpointApi.BASIC_MERKLED_API.name(), EndpointApi.BMAS.name());
filterDef.filterEndpoints = ImmutableList.of(EndpointApi.BASIC_MERKLED_API.name(), EndpointApi.BMAS.name(), EndpointApi.WS2P.name());
filterDef.currency = parameters.getCurrency();
// Default sort
......@@ -451,7 +451,7 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
List<Peer> result = new ArrayList<>();
// If less than 100 node, get it in ONE call
if (leaves.size() < 100) {
if (leaves.size() <= 2000) {
List<Peer> peers = networkRemoteService.getPeers(peer);
if (CollectionUtils.isNotEmpty(peers)) {
......@@ -496,9 +496,12 @@ public class NetworkServiceImpl extends BaseRemoteServiceImpl implements Network
NetworkPeers.Peer peer = networkRemoteService.getPeerLeaf(requestedPeer, leaf);
addEndpointsAsPeers(peer, result, leaf, filterEndpoints);
} catch(HttpNotFoundException | TechnicalException e) {
} catch(HttpNotFoundException hnfe) {
log.debug("Peer not found for leaf=" + leaf);
// skip
} catch(TechnicalException e) {
log.warn("Error while getting peer leaf=" + leaf, e.getMessage());
// skip
}
}
}
......
This diff is collapsed.
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.duniter</groupId>
<artifactId>duniter4j</artifactId>
<version>1.0.4-SNAPSHOT</version>
</parent>
<artifactId>duniter4j-es-assembly</artifactId>
<packaging>pom</packaging>
<name>Duniter4j :: ElasticSearch Assembly</name>
<description>Build a ElasticSearch releases with all Duniter4j plugins</description>
<properties>
<!-- bundle configuration -->
<bundlePrefix>duniter4j-es-${project.version}</bundlePrefix>
<!-- i18n configuration -->
<i18n.bundleOutputName>duniter4j-es-i18n</i18n.bundleOutputName>
<i18n.generateCsvFile>true</i18n.generateCsvFile>
<i18n.bundleCsvFile>
${maven.gen.dir}/resources/META-INF/${i18n.bundleOutputName}.csv
</i18n.bundleCsvFile>
<config.i18nBundleName>${i18n.bundleOutputName}</config.i18nBundleName>
<assembly.skip>false</assembly.skip>
</properties>
<dependencies>
<dependency>
<groupId>javax.websocket</groupId>
<artifactId>javax.websocket-api</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-client</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-server</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-container-grizzly-client</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-container-grizzly-server</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>${jna.version}</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
<version>${jna.version}</version>
<exclusions>
<exclusion>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<!-- unpack ES -->
<execution>
<id>unpack-elasticsearch</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.elasticsearch.distribution.zip</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
<type>zip</type>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/</outputDirectory>
<silent>true</silent>
<skip>${assembly.skip}</skip>
</configuration>
</execution>
<!-- unpack attachment plugin -->
<execution>
<id>unpack-mapper-attachments-plugin</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>mapper-attachments</artifactId>
<version>${elasticsearch.version}</version>
<type>zip</type>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/elasticsearch-${elasticsearch.version}/plugins/mapper-attachments</outputDirectory>
<silent>true</silent>
<skip>${assembly.skip}</skip>
</configuration>
</execution>
<!-- unpack ES core plugin -->
<execution>
<id>unpack-es-core-plugin</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.duniter</groupId>
<artifactId>duniter4j-es-core</artifactId>
<version>${project.version}</version>
<type>zip</type>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/elasticsearch-${elasticsearch.version}/plugins/duniter4j-es-core</outputDirectory>
<silent>true</silent>
<skip>${assembly.skip}</skip>
</configuration>
</execution>
<!-- unpack ES user plugin -->
<execution>
<id>unpack-es-user-plugin</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.duniter</groupId>
<artifactId>duniter4j-es-user</artifactId>
<version>${project.version}</version>
<type>zip</type>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/elasticsearch-${elasticsearch.version}/plugins/duniter4j-es-user</outputDirectory>
<silent>true</silent>
<skip>${assembly.skip}</skip>
</configuration>
</execution>
<!-- unpack ES subscription plugin -->
<execution>
<id>unpack-es-subscription-plugin</id>
<goals>
<goal>unpack</goal>
</goals>
<phase>prepare-package</phase>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.duniter</groupId>
<artifactId>duniter4j-es-subscription</artifactId>
<version>${project.version}</version>
<type>zip</type>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/elasticsearch-${elasticsearch.version}/plugins/duniter4j-es-subscription</outputDirectory>
<silent>true</silent>
<skip>${assembly.skip}</skip>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-assembly-plugin</artifactId>
<executions>
<execution>
<id>assembly-standalone</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<attach>true</attach>
<finalName>${bundlePrefix}</finalName>
<descriptors>
<descriptor>
${basedir}/src/main/assembly/standalone.xml
</descriptor>
</descriptors>
<skipAssembly>${assembly.skip}</skipAssembly>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<profiles>
<!-- use this profile to run the main class -->
<profile>
<id>run</id>
<activation>
<activeByDefault>false</activeByDefault>
</activation>
<build>
<defaultGoal>integration-test</defaultGoal>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-antrun-plugin</artifactId>
<dependencies>
<dependency>
<groupId>ant-contrib</groupId>
<artifactId>ant-contrib</artifactId>
<version>1.0b3</version>
<exclusions>
<exclusion>
<groupId>ant</groupId>
<artifactId>ant</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<executions>
<execution>
<id>install-duniter-plugin-jar</id>
<phase>pre-integration-test</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<target>
<ac:if xmlns:ac="antlib:net.sf.antcontrib">
<istrue value="${assembly.skip}" />
<!-- reuse standalone files -->
<then>
<delete failonerror="false">
<fileset dir="${run.es.home}/plugins" includes="**/duniter4j-*.jar" />
</delete>
<copy todir="${run.es.home}/plugins/duniter4j-es-core" overwrite="true">
<fileset dir="../duniter4j-core-client/target" includes="duniter4j-*${project.version}.jar">
</fileset>
<fileset dir="../duniter4j-core-shared/target" includes="duniter4j-*${project.version}.jar">
</fileset>
<fileset dir="../duniter4j-es-core/target" includes="duniter4j-*${project.version}.jar">
</fileset>
</copy>
<copy todir="${run.es.home}/plugins/duniter4j-es-core" overwrite="true">
<fileset dir="../duniter4j-es-core/target" includes="duniter4j-*${project.version}.jar">
</fileset>
</copy>
<copy todir="${run.es.home}/plugins/duniter4j-es-user" overwrite="true">
<fileset dir="../duniter4j-es-user/target" includes="duniter4j-*${project.version}.jar">
</fileset>
</copy>
<copy todir="${run.es.home}/plugins/duniter4j-es-subscription" overwrite="true">
<fileset dir="../duniter4j-es-subscription/target" includes="duniter4j-*${project.version}.jar">
</fileset>
</copy>
</then>
<else>
<delete dir="${project.build.directory}/${bundlePrefix}" />
<delete dir="${run.es.home}" />
<!-- Unzip standalone zip-->
<unzip src="${project.build.directory}/${bundlePrefix}-standalone.zip" dest="${project.build.directory}" overwrite="true">
</unzip>
<move file="${project.build.directory}/${bundlePrefix}" tofile="${run.es.home}" />
</else>
</ac:if>
<!-- Use files from src/test/es-home -->
<copy todir="${run.es.home}" overwrite="true">
<fileset dir="${project.basedir}/src/test/es-home" includes="**/*.*">
</fileset>
</copy>
</target>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>${log4j.version}</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>${jna.version}</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna-platform</artifactId>
<version>${jna.version}</version>
<exclusions>
<exclusion>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>javax.websocket</groupId>
<artifactId>javax.websocket-api</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-client</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-container-grizzly-client</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-server</artifactId>
<version>${tyrus.version}</version>
</dependency>
<dependency>
<groupId>org.glassfish.tyrus</groupId>
<artifactId>tyrus-container-grizzly-server</artifactId>
<version>${tyrus.version}</version>
</dependency>
</dependencies>
<executions>
<execution>
<id>run</id>
<goals>
<goal>java</goal>
</goals>
<phase>integration-test</phase>
<configuration>
<mainClass>org.elasticsearch.bootstrap.Elasticsearch</mainClass>
<arguments>
<argument>start</argument>
</arguments>
<includeProjectDependencies>false</includeProjectDependencies>
<includePluginDependencies>true</includePluginDependencies>
<systemProperties>
<systemProperty>
<key>es.path.home</key>
<value>${run.es.home}</value>
</systemProperty>
</systemProperties>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<properties>
<run.es.home>${project.build.directory}/es-run-home</run.es.home>
</properties>
</profile>
</profiles>
</project>
# Generated by org.codehaus.mojo.license.AddThirdPartyMojo
#-------------------------------------------------------------------------------
# Already used licenses in project :
# - ASL, version 2
# - Apache License 2.0
# - Apache License Version 2.0
# - BSD License
# - CC0 1.0 Universal
# - COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0
# - Eclipse Public License 1.0
# - General Public License (GPL) v3
# - Indiana University Extreme! Lab Software License, vesion 1.1.1
# - LGPL, version 2.1
# - Lesser General Public License (LGPL) v 3.0
# - Lesser General Public License (LPGL)
# - Lesser General Public License (LPGL) v 2.1
# - MIT License
# - New BSD License
# - Public Domain, per Creative Commons CC0
# - The Apache Software License, Version 2.0
#-------------------------------------------------------------------------------