From 9603aed3f20cf253df31e4a30e504ec3cd768171 Mon Sep 17 00:00:00 2001 From: blavenie <benoit.lavenier@e-is.pro> Date: Sat, 6 Aug 2016 12:50:20 +0200 Subject: [PATCH] - Update nuiton library (config and i18n) - Add missing i18n - Add REST action registry/record/_update - Run blockchain synchronization at node startup - Fix classpath issue for WebSocket --- README.md | 29 +++++ .../core/client/config/Configuration.java | 2 +- .../client/config/ConfigurationOption.java | 2 +- .../core/client/service/HttpServiceImpl.java | 25 ++-- .../bma/BlockchainRemoteServiceImpl.java | 2 + .../duniter4j-core-client_en_GB.properties | 6 + .../duniter4j-core-client_fr_FR.properties | 6 + duniter4j-elasticsearch/pom.xml | 122 ++++++++---------- .../main/assembly/config/elasticsearch.yml | 10 +- .../src/main/assembly/config/logging.yml | 5 + .../duniter/elasticsearch/PluginSettings.java | 6 +- .../elasticsearch/action/RestModule.java | 2 + .../RestRegistryRecordUpdateAction.java | 69 ++++++++++ .../elasticsearch/node/DuniterNode.java | 34 +++-- .../service/AbstractService.java | 50 ++++++- .../service/BlockchainService.java | 7 + .../elasticsearch/service/MarketService.java | 26 ++-- .../service/RegistryService.java | 35 +++-- .../javax.websocket.ContainerProvider | 1 + .../market-categories-bulk-insert.json | 20 +-- pom.xml | 10 +- 21 files changed, 332 insertions(+), 137 deletions(-) create mode 100644 duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/registry/RestRegistryRecordUpdateAction.java create mode 100644 duniter4j-elasticsearch/src/main/resources/META-INF/services/javax.websocket.ContainerProvider diff --git a/README.md b/README.md index 843c5929..deaf6a8c 100644 --- a/README.md +++ b/README.md @@ -202,3 +202,32 @@ $ mvn install -DskipTests -DperformRelease - Add an embedded [Cesium](https://www.github.com/duniter/cesium) inside the ElasticSearch plugin - Detect blockchain rollback + + +## Troubleshooting + +### Could not find an implementation class. + +Message: + +``` +java.lang.RuntimeException: java.lang.RuntimeException: Could not find an implementation class. + at org.duniter.core.util.websocket.WebsocketClientEndpoint.<init>(WebsocketClientEndpoint.java:56) + at org.duniter.core.client.service.bma.BlockchainRemoteServiceImpl.addNewBlockListener(BlockchainRemoteServiceImpl.java:545) + at org.duniter.elasticsearch.service.BlockchainService.listenAndIndexNewBlock(BlockchainService.java:106) +``` + +Cause: + +Plugin use Websocket to get notification from a Duniter nodes. The current library ([Tyrus](https://tyrus.java.net/)) is loaded throw java Service Loader, that need access to file `META-INF/services/javax.websocket.ContainerProvider` contains by Tyrus. +ElasticSearch use separated classloader, for each plugin, that disable access to META-INF resource. + +Solution : + +Move Tyrus libraries into elasticsearch `lib/` directory : + +``` + cd <ES_HOME> + mv plugins/duniter4j-elasticsearch/tyrus-*.jar lib + mv plugins/duniter4j-elasticsearch/javax.websocket-api-*.jar lib +``` \ No newline at end of file diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/config/Configuration.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/config/Configuration.java index a13b9fed..aa06a3a1 100644 --- a/duniter4j-core-client/src/main/java/org/duniter/core/client/config/Configuration.java +++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/config/Configuration.java @@ -29,7 +29,7 @@ import org.nuiton.config.ApplicationConfig; import org.nuiton.config.ApplicationConfigHelper; import org.nuiton.config.ApplicationConfigProvider; import org.nuiton.config.ArgumentsParserException; -import org.nuiton.util.version.Version; +import org.nuiton.version.Version; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/config/ConfigurationOption.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/config/ConfigurationOption.java index 77ae3141..0c743f17 100644 --- a/duniter4j-core-client/src/main/java/org/duniter/core/client/config/ConfigurationOption.java +++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/config/ConfigurationOption.java @@ -60,7 +60,7 @@ public enum ConfigurationOption implements ConfigOptionDef { I18N_DIRECTORY( "duniter4j.i18n.directory", n("duniter4j.config.option.i18n.directory.description"), - "${duniter4j.basedir}/i18n", + "${duniter4j.data.directory}/i18n", File.class), TMP_DIRECTORY( diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java index 4e7cdc89..4f1bcd23 100644 --- a/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java +++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/service/HttpServiceImpl.java @@ -41,6 +41,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; +import org.nuiton.i18n.I18n; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +57,6 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean private static final Logger log = LoggerFactory.getLogger(HttpServiceImpl.class); - private static final String USER_AGENT = "Android"; - public static final String URL_PEER_ALIVE = "/blockchain/parameters"; protected Integer baseTimeOut; @@ -190,24 +189,24 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean } case HttpStatus.SC_UNAUTHORIZED: case HttpStatus.SC_FORBIDDEN: - throw new TechnicalException("duniter4j.client.authentication"); + throw new TechnicalException(I18n.t("duniter4j.client.authentication")); case HttpStatus.SC_BAD_REQUEST: try { Error error = (Error)parseResponse(response, Error.class); throw new HttpBadRequestException(error); } catch(IOException e) { - throw new HttpBadRequestException("duniter4j.client.status" + response.getStatusLine().toString()); + throw new HttpBadRequestException(I18n.t("duniter4j.client.status", response.getStatusLine().toString())); } default: - throw new TechnicalException("duniter4j.client.status" + response.getStatusLine().toString()); + throw new TechnicalException(I18n.t("duniter4j.client.status", response.getStatusLine().toString())); } } catch (ConnectException e) { - throw new TechnicalException("duniter4j.client.core.connect", e); + throw new TechnicalException(I18n.t("duniter4j.client.core.connect", request.toString()), e); } catch (SocketTimeoutException e) { - throw new TechnicalException("duniter4j.client.core.timeout", e); + throw new TechnicalException(I18n.t("duniter4j.client.core.timeout"), e); } catch (IOException e) { throw new TechnicalException(e.getMessage(), e); @@ -272,10 +271,10 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean else { log.warn("Error while parsing JSON response", e); } - throw new JsonSyntaxException("ucoin.client.core.invalidResponse", e); + throw new JsonSyntaxException(I18n.t("duniter4j.client.core.invalidResponse"), e); } catch (Exception e) { - throw new TechnicalException("ucoin.client.core.invalidResponse", e); + throw new TechnicalException(I18n.t("duniter4j.client.core.invalidResponse"), e); } finally { if (content!= null) { @@ -285,7 +284,7 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean } if (result == null) { - throw new TechnicalException("ucoin.client.core.emptyResponse"); + throw new TechnicalException(I18n.t("duniter4j.client.core.emptyResponse")); } return result; @@ -318,14 +317,14 @@ public class HttpServiceImpl implements HttpService, Closeable, InitializingBean } case HttpStatus.SC_UNAUTHORIZED: case HttpStatus.SC_FORBIDDEN: - throw new TechnicalException("ucoin.client.authentication"); + throw new TechnicalException(I18n.t("duniter4j.client.authentication")); default: - throw new TechnicalException("ucoin.client.status" + response.getStatusLine().toString()); + throw new TechnicalException(I18n.t("duniter4j.client.status", response.getStatusLine().toString())); } } catch (ConnectException e) { - throw new TechnicalException("ucoin.client.core.connect", e); + throw new TechnicalException(I18n.t("duniter4j.client.core.connect"), e); } catch (IOException e) { throw new TechnicalException(e.getMessage(), e); diff --git a/duniter4j-core-client/src/main/java/org/duniter/core/client/service/bma/BlockchainRemoteServiceImpl.java b/duniter4j-core-client/src/main/java/org/duniter/core/client/service/bma/BlockchainRemoteServiceImpl.java index 149bfcdc..3375638a 100644 --- a/duniter4j-core-client/src/main/java/org/duniter/core/client/service/bma/BlockchainRemoteServiceImpl.java +++ b/duniter4j-core-client/src/main/java/org/duniter/core/client/service/bma/BlockchainRemoteServiceImpl.java @@ -539,6 +539,8 @@ public class BlockchainRemoteServiceImpl extends BaseRemoteServiceImpl implement peer.getHost(), peer.getPort())); + log.info(String.format("Starting to listen block from [%s]...", wsBlockURI.toString())); + // Get the websocket, or open new one if not exists WebsocketClientEndpoint wsClientEndPoint = blockWsEndPoints.get(wsBlockURI); if (wsClientEndPoint == null || wsClientEndPoint.isClosed()) { diff --git a/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_en_GB.properties b/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_en_GB.properties index 2f2972d9..c1f8c8c8 100644 --- a/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_en_GB.properties +++ b/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_en_GB.properties @@ -1,3 +1,9 @@ +duniter4j.client.authentication= +duniter4j.client.core.connect=Could not connect to Duniter node [%s] +duniter4j.client.core.emptyResponse= +duniter4j.client.core.invalidResponse= +duniter4j.client.core.timeout= +duniter4j.client.status= duniter4j.config= duniter4j.config.option.basedir.description= duniter4j.config.option.cache.directory.description= diff --git a/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_fr_FR.properties b/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_fr_FR.properties index 2f2972d9..dccc7d15 100644 --- a/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_fr_FR.properties +++ b/duniter4j-core-client/src/main/resources/i18n/duniter4j-core-client_fr_FR.properties @@ -1,3 +1,9 @@ +duniter4j.client.authentication= +duniter4j.client.core.connect=Echec de la connection au noeud Duniter [%s] +duniter4j.client.core.emptyResponse= +duniter4j.client.core.invalidResponse= +duniter4j.client.core.timeout= +duniter4j.client.status= duniter4j.config= duniter4j.config.option.basedir.description= duniter4j.config.option.cache.directory.description= diff --git a/duniter4j-elasticsearch/pom.xml b/duniter4j-elasticsearch/pom.xml index c6591943..d44be035 100644 --- a/duniter4j-elasticsearch/pom.xml +++ b/duniter4j-elasticsearch/pom.xml @@ -119,19 +119,6 @@ </resources> <plugins> - <plugin> - <artifactId>maven-jar-plugin</artifactId> - <configuration> - <archive> - <manifest> - <useUniqueVersions>false</useUniqueVersions> - <addClasspath>true</addClasspath> - <classpathPrefix>./lib/</classpathPrefix> - </manifest> - </archive> - </configuration> - </plugin> - <plugin> <groupId>org.nuiton.i18n</groupId> <artifactId>i18n-maven-plugin</artifactId> @@ -280,7 +267,7 @@ <activeByDefault>false</activeByDefault> </activation> <build> - <defaultGoal>package</defaultGoal> + <defaultGoal>integration-test</defaultGoal> <plugins> <plugin> <artifactId>maven-enforcer-plugin</artifactId> @@ -313,11 +300,11 @@ <artifactId>maven-dependency-plugin</artifactId> <executions> <execution> - <id>unpack-dependencies</id> + <id>unpack-elasticsearch</id> <goals> <goal>unpack</goal> </goals> - <phase>prepare-package</phase> + <phase>initialize</phase> <configuration> <artifactItems> <artifactItem> @@ -331,24 +318,37 @@ <silent>true</silent> </configuration> </execution> - </executions> - </plugin> - - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-dependency-plugin</artifactId> - <executions> + <execution> + <id>unpack-mapper-attachments-plugin</id> + <goals> + <goal>unpack</goal> + </goals> + <phase>initialize</phase> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.elasticsearch.plugin</groupId> + <artifactId>mapper-attachments</artifactId> + <version>${elasticsearch.version}</version> + <type>zip</type> + </artifactItem> + </artifactItems> + <outputDirectory>${es.home}/plugins/mapper-attachments</outputDirectory> + <silent>true</silent> + </configuration> + </execution> <execution> <id>copy-dependencies</id> - <phase>prepare-package</phase> + <phase>initialize</phase> <goals> <goal>copy-dependencies</goal> </goals> <configuration> - <outputDirectory>${project.build.directory}/es-home/plugins/${project.artifactId}</outputDirectory> + <outputDirectory>${es.home}/plugins/${project.artifactId}</outputDirectory> <excludeArtifactIds>jna,jackson-core,log4j,elasticsearch</excludeArtifactIds> <overWriteSnapshots>true</overWriteSnapshots> <silent>true</silent> + <includeScope>runtime</includeScope> </configuration> </execution> </executions> @@ -372,22 +372,25 @@ </dependencies> <executions> <execution> - <id>unpack-elasticsearch-binaries</id> - <phase>initialize</phase> + <id>install-elasticsearch-binaries</id> + <phase>generate-resources</phase> <goals> <goal>run</goal> </goals> <configuration> <target> - <copy todir="${es.home}"> - <fileset dir="${project.build.directory}/elasticsearch-${elasticsearch.version}"> - </fileset> - </copy> - <chmod perm="u+x"> - <fileset dir="${es.home}/bin"> - <include name="elasticsearch"/> + <!-- Change execution right --> + <chmod perm="ug+x"> + <fileset dir="${es.home}"> + <include name="bin/elasticsearch"/> + <include name="bin/plugin"/> </fileset> </chmod> + <chmod perm="ug+rw"> + <fileset dir="${es.home}/lib"/> + </chmod> + + <!-- Override config files --> <copy todir="${es.home}/config" overwrite="true" filtering="true"> @@ -397,27 +400,6 @@ </target> </configuration> </execution> - <execution> - <id>download-attachment-mapper-plugin</id> - <phase>initialize</phase> - <goals> - <goal>run</goal> - </goals> - <configuration> - <target> - - <!-- download attachment plugin --> - <get src="https://download.elastic.co/elasticsearch/release/org/elasticsearch/plugin/mapper-attachments/${elasticsearch.version}/mapper-attachments-${elasticsearch.version}.zip" - dest="${project.build.directory}/mapper-attachments-${elasticsearch.version}.zip" - verbose="false" - usetimestamp="true"/> - <unzip src="${project.build.directory}/mapper-attachments-${elasticsearch.version}.zip" - dest="${es.home}/plugins/mapper-attachments" - overwrite="true"> - </unzip> - </target> - </configuration> - </execution> <execution> <id>download-cesium</id> <phase>initialize</phase> @@ -427,7 +409,7 @@ <configuration> <target> - <!-- download cesium > + <!-- download cesium --> <get src="${cesium.download.url}" dest="${project.build.directory}/cesium-web-${cesium.version}.zip" verbose="false" @@ -435,18 +417,26 @@ <unzip src="${project.build.directory}/cesium-web-${cesium.version}.zip" dest="${duniter4j.plugin.directory}/_site" overwrite="true"> - </unzip--> + </unzip> </target> </configuration> </execution> <execution> <id>install-duniter-plugin</id> - <phase>prepare-package</phase> + <phase>pre-integration-test</phase> <goals> <goal>run</goal> </goals> <configuration> <target> + + <!-- Copy plugin main jar --> + <copy todir="${duniter4j.plugin.directory}"> + <fileset dir="${project.build.directory}"> + <include name="${project.artifactId}-${project.version}.${project.packaging}"/> + </fileset> + </copy> + <!-- Copy descriptor file and security files --> <copy todir="${duniter4j.plugin.directory}" filtering="true"> @@ -455,6 +445,13 @@ <include name="plugin-security.policy"/> </fileset> </copy> + <!-- Copy main libs --> + <move todir="${es.home}/lib"> + <fileset dir="${duniter4j.plugin.directory}"> + <include name="tyrus-*.jar"/> + <include name="javax.websocket-api-*.jar"/> + </fileset> + </move> <!-- Remove redundant lib in duniter plugin --> <ac:for param="file" xmlns:ac="antlib:net.sf.antcontrib"> @@ -480,11 +477,6 @@ <include name="guava-*.jar"/> </fileset> </delete> - <copy todir="${duniter4j.plugin.directory}"> - <fileset dir="${project.build.directory}"> - <include name="${project.artifactId}-${project.version}.${project.packaging}"/> - </fileset> - </copy> </target> </configuration> @@ -501,7 +493,7 @@ <goals> <goal>exec</goal> </goals> - <phase>package</phase> + <phase>integration-test</phase> <configuration> <executable>${es.home}/bin/elasticsearch</executable> <workingDirectory>${es.home}</workingDirectory> @@ -517,7 +509,7 @@ <exec.classpathScope>runtime</exec.classpathScope> <duniter4j.log.file>${project.build.directory}/exec.log</duniter4j.log.file> - <es.home>${project.build.directory}/es-home</es.home> + <es.home>${project.build.directory}/elasticsearch-${elasticsearch.version}</es.home> <duniter4j.basedir>${es.home}</duniter4j.basedir> <duniter4j.plugin.directory>${es.home}/plugins/${project.artifactId}</duniter4j.plugin.directory> <es.http.cors.allow-origin>*</es.http.cors.allow-origin> diff --git a/duniter4j-elasticsearch/src/main/assembly/config/elasticsearch.yml b/duniter4j-elasticsearch/src/main/assembly/config/elasticsearch.yml index 4d083386..e5f25176 100644 --- a/duniter4j-elasticsearch/src/main/assembly/config/elasticsearch.yml +++ b/duniter4j-elasticsearch/src/main/assembly/config/elasticsearch.yml @@ -102,13 +102,19 @@ security.manager.enabled: false #duniter.disable: true -duniter.host: cgeek.fr -duniter.port: 9330 +#duniter.host: cgeek.fr +#duniter.port: 9330 + +duniter.host: 192.168.0.5 +duniter.port: 9201 duniter.string.analyzer: french #duniter.indices.reload: true +# Should synchronize node blockchain ? +duniter.blockchain.sync.enable: true + #duniter.dev.enable: true #script.groovy.sandbox.enabled: true diff --git a/duniter4j-elasticsearch/src/main/assembly/config/logging.yml b/duniter4j-elasticsearch/src/main/assembly/config/logging.yml index e183340c..d99a4f56 100644 --- a/duniter4j-elasticsearch/src/main/assembly/config/logging.yml +++ b/duniter4j-elasticsearch/src/main/assembly/config/logging.yml @@ -18,6 +18,11 @@ logger: org.duniter: INFO + #org.duniter.elasticsearch: DEBUG + + org.nuiton.i18n: WARN + org.nuiton.config: WARN + # gateway #gateway: DEBUG #index.gateway: DEBUG diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/PluginSettings.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/PluginSettings.java index 07688f0e..1c9d4651 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/PluginSettings.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/PluginSettings.java @@ -169,6 +169,10 @@ public class PluginSettings extends AbstractLifecycleComponent<PluginSettings> { return settings.getAsBoolean("duniter.indices.reload", false); } + public boolean enableBlockchainSync() { + return settings.getAsBoolean("duniter.blockchain.sync.enable", false); + } + public File getTempDirectory() { return Configuration.instance().getTempDirectory(); } @@ -201,7 +205,7 @@ public class PluginSettings extends AbstractLifecycleComponent<PluginSettings> { // init i18n // --------------------------------------------------------------------// - File i18nDirectory = new File(clientConfig.getDataDirectory(), "i18n"); + File i18nDirectory = clientConfig.getI18nDirectory(); if (i18nDirectory.exists()) { // clean i18n cache FileUtils.cleanDirectory(i18nDirectory); diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/RestModule.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/RestModule.java index ab35ea1c..104bfe79 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/RestModule.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/RestModule.java @@ -29,6 +29,7 @@ import org.duniter.elasticsearch.action.market.RestMarketCommentUpdateAction; import org.duniter.elasticsearch.action.market.RestMarketRecordIndexAction; import org.duniter.elasticsearch.action.market.RestMarketRecordUpdateAction; import org.duniter.elasticsearch.action.registry.RestRegistryRecordIndexAction; +import org.duniter.elasticsearch.action.registry.RestRegistryRecordUpdateAction; import org.duniter.elasticsearch.action.security.RestSecurityAuthAction; import org.duniter.elasticsearch.action.security.RestSecurityGetChallengeAction; import org.duniter.elasticsearch.action.user.RestUserProfileIndexAction; @@ -51,6 +52,7 @@ public class RestModule extends AbstractModule implements Module { // Registry bind(RestRegistryRecordIndexAction.class).asEagerSingleton(); + bind(RestRegistryRecordUpdateAction.class).asEagerSingleton(); // User bind(RestUserProfileIndexAction.class).asEagerSingleton(); diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/registry/RestRegistryRecordUpdateAction.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/registry/RestRegistryRecordUpdateAction.java new file mode 100644 index 00000000..45c480c1 --- /dev/null +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/action/registry/RestRegistryRecordUpdateAction.java @@ -0,0 +1,69 @@ +package org.duniter.elasticsearch.action.registry; + +/* + * #%L + * duniter4j-elasticsearch-plugin + * %% + * Copyright (C) 2014 - 2016 EIS + * %% + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program. If not, see + * <http://www.gnu.org/licenses/gpl-3.0.html>. + * #L% + */ + +import org.duniter.core.exception.BusinessException; +import org.duniter.elasticsearch.exception.DuniterElasticsearchException; +import org.duniter.elasticsearch.rest.XContentThrowableRestResponse; +import org.duniter.elasticsearch.service.MarketService; +import org.duniter.elasticsearch.service.RegistryService; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.logging.ESLogger; +import org.elasticsearch.common.logging.ESLoggerFactory; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.rest.*; + +import static org.elasticsearch.rest.RestRequest.Method.POST; +import static org.elasticsearch.rest.RestStatus.OK; + +public class RestRegistryRecordUpdateAction extends BaseRestHandler { + + private static final ESLogger log = ESLoggerFactory.getLogger(RestRegistryRecordUpdateAction.class.getName()); + + private RegistryService service; + + @Inject + public RestRegistryRecordUpdateAction(Settings settings, RestController controller, Client client, RegistryService service) { + super(settings, controller, client); + controller.registerHandler(POST, "/registry/record/{id}/_update", this); + this.service = service; + } + + @Override + protected void handleRequest(final RestRequest request, RestChannel restChannel, Client client) throws Exception { + String id = request.param("id"); + try { + service.updateRecordFromJson(request.content().toUtf8(), id); + restChannel.sendResponse(new BytesRestResponse(OK, id)); + } + catch(DuniterElasticsearchException | BusinessException e) { + log.error(e.getMessage(), e); + restChannel.sendResponse(new XContentThrowableRestResponse(request, e)); + } + catch(Exception e) { + log.error(e.getMessage(), e); + } + } + +} \ No newline at end of file diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/node/DuniterNode.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/node/DuniterNode.java index d0b8745c..80be1d95 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/node/DuniterNode.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/node/DuniterNode.java @@ -22,7 +22,11 @@ package org.duniter.elasticsearch.node; * #L% */ +import org.duniter.core.client.model.bma.BlockchainBlock; +import org.duniter.core.client.model.bma.gson.GsonUtils; import org.duniter.core.client.model.local.Peer; +import org.duniter.core.client.service.bma.BlockchainRemoteService; +import org.duniter.core.util.websocket.WebsocketClientEndpoint; import org.duniter.elasticsearch.PluginSettings; import org.duniter.elasticsearch.service.*; import org.duniter.elasticsearch.threadpool.ThreadPool; @@ -53,6 +57,8 @@ public class DuniterNode extends AbstractLifecycleComponent<DuniterNode> { protected void doStart() { threadPool.scheduleOnStarted(() -> { createIndices(); + + synchronize(); }); } @@ -69,20 +75,17 @@ public class DuniterNode extends AbstractLifecycleComponent<DuniterNode> { protected void createIndices() { boolean reloadIndices = pluginSettings.reloadIndices(); - Peer peer = pluginSettings.checkAndGetPeer(); + if (reloadIndices) { if (logger.isInfoEnabled()) { logger.info("Reloading all Duniter indices..."); } injector.getInstance(RegistryService.class) .deleteIndex() - .createIndexIfNotExists() - .fillRecordCategories() - .indexCurrencyFromPeer(peer); + .createIndexIfNotExists(); injector.getInstance(MarketService.class) .deleteIndex() - .createIndexIfNotExists() - .fillRecordCategories(); + .createIndexIfNotExists(); injector.getInstance(MessageService.class) .deleteIndex() .createIndexIfNotExists(); @@ -95,10 +98,6 @@ public class DuniterNode extends AbstractLifecycleComponent<DuniterNode> { .deleteIndex() .createIndexIfNotExists(); - injector.getInstance(BlockchainService.class) - .indexLastBlocks(peer); - - if (logger.isInfoEnabled()) { logger.info("Reloading all Duniter indices... [OK]"); } @@ -118,6 +117,21 @@ public class DuniterNode extends AbstractLifecycleComponent<DuniterNode> { logger.info("Checking Duniter indices... [OK]"); } } + } + + protected void synchronize() { + if (pluginSettings.enableBlockchainSync()) { + + Peer peer = pluginSettings.checkAndGetPeer(); + + // Index (or refresh) node's currency + injector.getInstance(RegistryService.class).indexCurrencyFromPeer(peer); + // Index blocks (and listen if new block appear) + injector.getInstance(BlockchainService.class) + //.indexLastBlocks(peer) + .listenAndIndexNewBlock(peer); + + } } } diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/AbstractService.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/AbstractService.java index d0610879..0e29f0c7 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/AbstractService.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/AbstractService.java @@ -56,6 +56,8 @@ import org.elasticsearch.common.xcontent.XContentFactory; import java.io.*; import java.util.Objects; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Created by Benoit on 08/04/2015. @@ -187,6 +189,10 @@ public abstract class AbstractService implements Bean { } protected void bulkFromClasspathFile(String classpathFile, String indexName, String indexType) { + bulkFromClasspathFile(classpathFile, indexName, indexType, null); + } + + protected void bulkFromClasspathFile(String classpathFile, String indexName, String indexType, StringReaderHandler handler) { InputStream is = null; try { is = getClass().getClassLoader().getResourceAsStream(classpathFile); @@ -194,7 +200,7 @@ public abstract class AbstractService implements Bean { throw new TechnicalException(String.format("Could not retrieve data file [%s] need to fill index [%s]: ", classpathFile, indexName)); } - bulkFromStream(is, indexName, indexType); + bulkFromStream(is, indexName, indexType, handler); } finally { if (is != null) { @@ -209,13 +215,17 @@ public abstract class AbstractService implements Bean { } protected void bulkFromFile(File file, String indexName, String indexType) { + bulkFromFile(file, indexName, indexType, null); + } + + protected void bulkFromFile(File file, String indexName, String indexType, StringReaderHandler handler) { Preconditions.checkNotNull(file); Preconditions.checkArgument(file.exists()); InputStream is = null; try { is = new BufferedInputStream(new FileInputStream(file)); - bulkFromStream(is, indexName, indexType); + bulkFromStream(is, indexName, indexType, handler); } catch(FileNotFoundException e) { throw new TechnicalException(String.format("[%s] Could not find file %s", indexName, file.getPath()), e); @@ -233,6 +243,10 @@ public abstract class AbstractService implements Bean { } protected void bulkFromStream(InputStream is, String indexName, String indexType) { + bulkFromStream(is, indexName, indexType, null); + } + + protected void bulkFromStream(InputStream is, String indexName, String indexType, StringReaderHandler handler) { Preconditions.checkNotNull(is); BulkRequest bulkRequest = Requests.bulkRequest(); @@ -244,10 +258,14 @@ public abstract class AbstractService implements Bean { String line = br.readLine(); StringBuilder builder = new StringBuilder(); while(line != null) { + line = line.trim(); if (StringUtils.isNotBlank(line)) { if (logger.isTraceEnabled()) { logger.trace(String.format("[%s] Add to bulk: %s", indexName, line)); } + if (handler != null) { + line = handler.onReadLine(line.trim()); + } builder.append(line).append('\n'); } line = br.readLine(); @@ -276,4 +294,32 @@ public abstract class AbstractService implements Bean { throw new TechnicalException(String.format("[%s] Error while inserting rows into %s", indexName, indexType), e); } } + + public interface StringReaderHandler { + + String onReadLine(String line); + } + + public class AddSequenceAttributeHandler implements StringReaderHandler { + private int order; + private final String attributeName; + private final Pattern filterPattern; + public AddSequenceAttributeHandler(String attributeName, String filterRegex, int startValue) { + this.order = startValue; + this.attributeName = attributeName; + this.filterPattern = Pattern.compile(filterRegex); + } + + @Override + public String onReadLine(String line) { + // add 'order' field into + if (filterPattern.matcher(line).matches()) { + return String.format("%s, \"%s\": %d}", + line.substring(0, line.length()-1), + attributeName, + order++); + } + return line; + } + } } diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/BlockchainService.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/BlockchainService.java index 7d654954..db71cb97 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/BlockchainService.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/BlockchainService.java @@ -102,6 +102,13 @@ public class BlockchainService extends AbstractService { this.registryService = registryService; } + public BlockchainService listenAndIndexNewBlock(Peer peer){ + blockchainRemoteService.addNewBlockListener(peer, message -> { + indexBlockAsJson(peer, message, true /*refresh*/, true /*wait*/); + }); + return this; + } + public BlockchainService indexLastBlocks(Peer peer) { return indexLastBlocks(peer, new ProgressionModelImpl()); } diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/MarketService.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/MarketService.java index 417b4207..6fe07d89 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/MarketService.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/MarketService.java @@ -25,20 +25,13 @@ package org.duniter.elasticsearch.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; -import com.google.gson.JsonSyntaxException; -import org.duniter.core.client.model.elasticsearch.DeleteRecord; -import org.duniter.core.client.model.elasticsearch.Record; import org.duniter.core.client.service.bma.WotRemoteService; import org.duniter.core.exception.TechnicalException; import org.duniter.core.service.CryptoService; import org.duniter.elasticsearch.PluginSettings; -import org.duniter.elasticsearch.exception.InvalidFormatException; -import org.duniter.elasticsearch.exception.InvalidSignatureException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.Client; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; @@ -46,7 +39,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import java.io.IOException; -import java.util.Set; /** * Created by Benoit on 30/03/2015. @@ -89,6 +81,9 @@ public class MarketService extends AbstractService { try { if (!existsIndex(INDEX)) { createIndex(); + + // Fill categories + fillRecordCategories(); } } catch(JsonProcessingException e) { @@ -103,7 +98,7 @@ public class MarketService extends AbstractService { * @throws JsonProcessingException */ public MarketService createIndex() throws JsonProcessingException { - logger.info(String.format("Creating index [%s/%s]", INDEX, RECORD_CATEGORY_TYPE)); + logger.info(String.format("Creating index [%s]", INDEX)); CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(INDEX); Settings indexSettings = Settings.settingsBuilder() @@ -209,13 +204,17 @@ public class MarketService extends AbstractService { .execute().actionGet(); } - public void fillRecordCategories() { + public MarketService fillRecordCategories() { if (logger.isDebugEnabled()) { logger.debug(String.format("[%s/%s] Fill data", INDEX, RECORD_CATEGORY_TYPE)); } // Insert categories - bulkFromClasspathFile(CATEGORIES_BULK_CLASSPATH_FILE, INDEX, RECORD_CATEGORY_TYPE); + bulkFromClasspathFile(CATEGORIES_BULK_CLASSPATH_FILE, INDEX, RECORD_CATEGORY_TYPE, + // Add order attribute (auto incremented) + new AddSequenceAttributeHandler("order", "\\{.*\"name\".*\\}", 1)); + + return this; } /* -- Internal methods -- */ @@ -231,6 +230,11 @@ public class MarketService extends AbstractService { .field("type", "string") .endObject() + // order + .startObject("order") + .field("type", "integer") + .endObject() + // description /*.startObject("description") .field("type", "string") diff --git a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/RegistryService.java b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/RegistryService.java index ceb4aa2b..9f595ec0 100644 --- a/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/RegistryService.java +++ b/duniter4j-elasticsearch/src/main/java/org/duniter/elasticsearch/service/RegistryService.java @@ -25,10 +25,8 @@ package org.duniter.elasticsearch.service; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; -import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.gson.JsonSyntaxException; import org.apache.commons.lang3.ArrayUtils; @@ -36,7 +34,6 @@ import org.duniter.core.client.model.bma.BlockchainBlock; import org.duniter.core.client.model.bma.BlockchainParameters; import org.duniter.core.client.model.bma.gson.GsonUtils; import org.duniter.core.client.model.elasticsearch.Currency; -import org.duniter.core.client.model.elasticsearch.Record; import org.duniter.core.client.model.local.Peer; import org.duniter.core.client.service.bma.BlockchainRemoteService; import org.duniter.core.client.service.bma.WotRemoteService; @@ -47,7 +44,6 @@ import org.duniter.core.util.StringUtils; import org.duniter.elasticsearch.PluginSettings; import org.duniter.elasticsearch.exception.AccessDeniedException; import org.duniter.elasticsearch.exception.DuplicateIndexIdException; -import org.duniter.elasticsearch.exception.InvalidFormatException; import org.duniter.elasticsearch.exception.InvalidSignatureException; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -69,7 +65,6 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Objects; -import java.util.Set; /** * Created by Benoit on 30/03/2015. @@ -103,6 +98,8 @@ public class RegistryService extends AbstractService { try { if (!existsIndex(INDEX)) { createIndex(); + + fillRecordCategories(); } } catch(JsonProcessingException e) { @@ -148,7 +145,9 @@ public class RegistryService extends AbstractService { } // Insert categories - bulkFromClasspathFile(CATEGORIES_BULK_CLASSPATH_FILE, INDEX, RECORD_CATEGORY_TYPE); + bulkFromClasspathFile(CATEGORIES_BULK_CLASSPATH_FILE, INDEX, RECORD_CATEGORY_TYPE, + // Add order attribute + new AddSequenceAttributeHandler("order", "\\{.*\"name\".*\\}", 1)); return this; } @@ -179,6 +178,23 @@ public class RegistryService extends AbstractService { return response.getId(); } + public void updateRecordFromJson(String recordJson, String id) { + + JsonNode actualObj = readAndVerifyIssuerSignature(recordJson); + String issuer = getIssuer(actualObj); + + // Check same document issuer + checkSameDocumentIssuer(INDEX, RECORD_TYPE, id, issuer); + + if (logger.isDebugEnabled()) { + logger.debug(String.format("Updating market record [%s] from issuer [%s]", id, issuer.substring(0, 8))); + } + + client.prepareUpdate(INDEX, RECORD_TYPE, id) + .setDoc(recordJson) + .execute().actionGet(); + } + public void insertRecordFromBulkFile(File bulkFile) { if (logger.isDebugEnabled()) { @@ -211,13 +227,6 @@ public class RegistryService extends AbstractService { indexCurrency(result); - // Index the first block - // FIXME : attention au dependence circulaire : cela devrait plutot etre fait à l'exetrieure e registry - // par exemple dans l'action REST - //blockBlockchainService.createIndexIfNotExists(parameters.getCurrency()); - //blockBlockchainService.indexBlock(firstBlock, false); - //blockBlockchainService.indexCurrentBlock(firstBlock, true); - return result; } diff --git a/duniter4j-elasticsearch/src/main/resources/META-INF/services/javax.websocket.ContainerProvider b/duniter4j-elasticsearch/src/main/resources/META-INF/services/javax.websocket.ContainerProvider new file mode 100644 index 00000000..3b4d294e --- /dev/null +++ b/duniter4j-elasticsearch/src/main/resources/META-INF/services/javax.websocket.ContainerProvider @@ -0,0 +1 @@ +org.glassfish.tyrus.client.ClientManager \ No newline at end of file diff --git a/duniter4j-elasticsearch/src/main/resources/market-categories-bulk-insert.json b/duniter4j-elasticsearch/src/main/resources/market-categories-bulk-insert.json index c19ddf13..5f0e335e 100644 --- a/duniter4j-elasticsearch/src/main/resources/market-categories-bulk-insert.json +++ b/duniter4j-elasticsearch/src/main/resources/market-categories-bulk-insert.json @@ -1,10 +1,10 @@ { "index": { "_id": "cat71"}} -{ "name": "EMPLOI" , "parent": null} +{ "name": "Emploi" , "parent": null} { "index": { "_id": "cat33"}} { "name": "Offres d'emploi", "parent": "cat33" } { "index": { "_id": "cat1" }} -{ "name": "VEHICULES" , "parent": null} +{ "name": "Véhicules" , "parent": null} { "index": { "_id": "cat2" }} { "name": "Voitures" , "parent": "cat2" } { "index": { "_id": "cat3" }} @@ -25,7 +25,7 @@ { "name": "Equipement Nautisme" , "parent": "cat3" } { "index": { "_id": "cat8" }} -{ "name": "IMMOBILIER" , "parent": null} +{ "name": "Immobilier" , "parent": null} { "index": { "_id": "cat9" }} { "name": "Ventes immobilières" , "parent": "cat8" } { "index": { "_id": "cat10" }} @@ -36,7 +36,7 @@ { "name": "Bureaux & Commerces" , "parent": "cat8" } { "index": { "_id": "cat66" }} -{ "name": "VACANCES" , "parent": null} +{ "name": "Vacances" , "parent": null} { "index": { "_id": "cat12" }} { "name": "Locations & Gîtes" , "parent": "cat66" } { "index": { "_id": "cat67" }} @@ -49,7 +49,7 @@ { "name": "Hébergements insolites" , "parent": "cat66" } { "index": { "_id": "cat14" }} -{ "name": "MULTIMEDIA" , "parent": null} +{ "name": "Multimédia" , "parent": null} { "index": { "_id": "cat15" }} { "name": "Informatique" , "parent": "cat14" } { "index": { "_id": "cat43" }} @@ -60,7 +60,7 @@ { "name": "Téléphonie" , "parent": "cat14" } { "index": { "_id": "cat18" }} -{ "name": "MAISON" , "parent": null} +{ "name": "Maison" , "parent": null} { "index": { "_id": "cat19" }} { "name": "Ameublement" , "parent": "cat18" } { "index": { "_id": "cat20" }} @@ -89,7 +89,7 @@ { "name": "Vêtements bébé" , "parent": "cat18" } { "index": { "_id": "cat24" }} -{ "name": "LOISIRS" , "parent": null} +{ "name": "Loisirs" , "parent": null} { "index": { "_id": "cat25" }} { "name": "DVD / Films" , "parent": "cat24" } { "index": { "_id": "cat26" }} @@ -112,7 +112,7 @@ { "name": "Vins & Gastronomie" , "parent": "cat24" } { "index": { "_id": "cat56" }} -{ "name": "MATERIEL PROFESSIONNEL" , "parent": null} +{ "name": "Matériel professionnel" , "parent": null} { "index": { "_id": "cat57" }} { "name": "Matériel Agricole" , "parent": "cat56" } { "index": { "_id": "cat58" }} @@ -133,7 +133,7 @@ { "name": "Matériel Médical" , "parent": "cat56" } { "index": { "_id": "cat31" }} -{ "name": "SERVICES" , "parent": null} +{ "name": "Services" , "parent": null} { "index": { "_id": "cat34" }} { "name": "Prestations de services" , "parent": "cat31" } { "index": { "_id": "cat35" }} @@ -146,6 +146,6 @@ { "name": "Covoiturage" , "parent": "cat31" } { "index": { "_id": "cat37" }} -{ "name": "DIVERS" , "parent": null} +{ "name": "Divers" , "parent": null} { "index": { "_id": "cat38" }} { "name": "Autres" , "parent": "cat37" } diff --git a/pom.xml b/pom.xml index 0ec59cf8..6b88c74d 100644 --- a/pom.xml +++ b/pom.xml @@ -28,8 +28,8 @@ <cesium.version>0.1.26</cesium.version> - <nuitonConfigVersion>3.0-rc-2</nuitonConfigVersion> - <nuitonI18nVersion>3.3</nuitonI18nVersion> + <nuitonConfigVersion>3.0-rc-4</nuitonConfigVersion> + <nuitonI18nVersion>3.5</nuitonI18nVersion> <!-- UI versions --> <spring.version>4.2.1.RELEASE</spring.version> @@ -430,12 +430,6 @@ <version>1.0</version> </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <version>1.8</version> - </plugin> - <plugin> <artifactId>maven-changes-plugin</artifactId> <version>2.11</version> -- GitLab