Skip to content
Snippets Groups Projects
Commit 99c72b49 authored by Benoit Lavenier's avatar Benoit Lavenier
Browse files

- add index for invitations

- update group index: remove group name (use attribute 'id' as ES id)
parent 88587f8c
No related branches found
No related tags found
No related merge requests found
Showing
with 429 additions and 47 deletions
......@@ -27,24 +27,14 @@ package org.duniter.core.client.model.elasticsearch;
*/
public class UserGroup extends Record {
public static final String PROPERTY_NAME="name";
public static final String PROPERTY_TITLE="title";
public static final String PROPERTY_DESCRIPTION="description";
public static final String PROPERTY_CREATION_TIME="creationTime";
private String name;
private String title;
private String description;
private Long creationTime;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getTitle() {
return title;
}
......
......@@ -138,8 +138,8 @@ duniter.keyring.password: def
# Enable security, to disable HTTP access to the default ES admin API
#
#duniter.security.enable: false
duniter.security.enable: true
duniter.security.enable: false
#duniter.security.enable: true
#
# Security token prefix (default: 'duniter-')
#
......@@ -153,9 +153,9 @@ duniter.security.enable: true
#
# Should synchronize data using P2P
#
duniter.data.sync.enable: false
#duniter.data.sync.host: data.duniter.fr
#duniter.data.sync.port: 80
duniter.data.sync.enable: true
duniter.data.sync.host: data.gtest.duniter.fr
duniter.data.sync.port: 80
# ---------------------------------- Duniter4j SMTP server -------------------------
#
......
......@@ -237,7 +237,6 @@ public abstract class AbstractService implements Bean {
}
}
protected String getIssuer(JsonNode actualObj) {
return getMandatoryField(actualObj, Record.PROPERTY_ISSUER).asText();
}
......@@ -259,7 +258,7 @@ public abstract class AbstractService implements Bean {
Map<String, Object> fields = getFieldsById(index, type, docId, fieldNames);
if (MapUtils.isEmpty(fields)) throw new NotFoundException(String.format("Document [%s/%s/%s] not exists.", index, type, docId));
Arrays.stream(fieldNames).forEach((fieldName) -> {
if (!fields.containsKey(fieldName)) throw new NotFoundException(String.format("Document [%s/%s/%s] should have the madatory field [%s].", index, type, docId, fieldName));
if (!fields.containsKey(fieldName)) throw new NotFoundException(String.format("Document [%s/%s/%s] should have the mandatory field [%s].", index, type, docId, fieldName));
});
return fields;
}
......
......@@ -172,6 +172,7 @@ public abstract class AbstractSynchroService extends AbstractService {
JsonNode node;
try {
HttpPost httpPost = new HttpPost(httpService.getPath(peer, fromIndex, fromType, "_search"));
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.setEntity(new ByteArrayEntity(bos.bytes().array()));
if (logger.isDebugEnabled()) {
logger.debug(String.format("[%s] [%s/%s] Sending POST request: %s", peer, fromIndex, fromType, new String(bos.bytes().array())));
......
......@@ -106,6 +106,9 @@ public class PluginInit extends AbstractLifecycleComponent<PluginInit> {
injector.getInstance(GroupService.class)
.deleteIndex()
.createIndexIfNotExists();
injector.getInstance(UserInvitationService.class)
.deleteIndex()
.createIndexIfNotExists();
if (logger.isInfoEnabled()) {
logger.info("Reloading all Duniter indices... [OK]");
......@@ -119,6 +122,7 @@ public class PluginInit extends AbstractLifecycleComponent<PluginInit> {
injector.getInstance(UserService.class).createIndexIfNotExists();
injector.getInstance(MessageService.class).createIndexIfNotExists();
injector.getInstance(GroupService.class).createIndexIfNotExists();
injector.getInstance(UserInvitationService.class).createIndexIfNotExists();
if (logger.isInfoEnabled()) {
logger.info("Checking Duniter indices... [OK]");
......
......@@ -45,6 +45,8 @@ public enum UserEventCodes {
CERT_RECEIVED,
// Message
MESSAGE_RECEIVED
MESSAGE_RECEIVED,
// Invitation
INVITATION_TO_CERTIFY
}
......@@ -25,6 +25,8 @@ package org.duniter.elasticsearch.user.rest;
import org.duniter.elasticsearch.user.rest.group.RestGroupIndexAction;
import org.duniter.elasticsearch.user.rest.group.RestGroupUpdateAction;
import org.duniter.elasticsearch.user.rest.history.RestHistoryDeleteIndexAction;
import org.duniter.elasticsearch.user.rest.invitation.RestInvitationCertificationIndexAction;
import org.duniter.elasticsearch.user.rest.invitation.RestInvitationCertificationMarkAsReadAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageInboxIndexAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageInboxMarkAsReadAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageOutboxIndexAction;
......@@ -61,6 +63,10 @@ public class RestModule extends AbstractModule implements Module {
bind(RestMessageOutboxIndexAction.class).asEagerSingleton();
bind(RestMessageInboxMarkAsReadAction.class).asEagerSingleton();
// Invitation
bind(RestInvitationCertificationIndexAction.class).asEagerSingleton();
bind(RestInvitationCertificationMarkAsReadAction.class).asEagerSingleton();
// Backward compatibility
{
// message/record
......
package org.duniter.elasticsearch.user.rest.invitation;
/*
* #%L
* duniter4j-elasticsearch-plugin
* %%
* Copyright (C) 2014 - 2016 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import org.duniter.elasticsearch.rest.AbstractRestPostIndexAction;
import org.duniter.elasticsearch.rest.security.RestSecurityController;
import org.duniter.elasticsearch.user.service.UserInvitationService;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestController;
public class RestInvitationCertificationIndexAction extends AbstractRestPostIndexAction {
@Inject
public RestInvitationCertificationIndexAction(Settings settings, RestController controller, Client client,
RestSecurityController securityController,
final UserInvitationService service) {
super(settings, controller, client, securityController,
UserInvitationService.INDEX,
UserInvitationService.CERTIFICATION_TYPE,
json -> service.indexCertificationInvitationFromJson(json));
}
}
\ No newline at end of file
package org.duniter.elasticsearch.user.rest.invitation;
/*
* #%L
* duniter4j-elasticsearch-plugin
* %%
* Copyright (C) 2014 - 2016 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import org.duniter.elasticsearch.rest.AbstractRestPostMarkAsReadAction;
import org.duniter.elasticsearch.rest.security.RestSecurityController;
import org.duniter.elasticsearch.user.service.UserInvitationService;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestController;
public class RestInvitationCertificationMarkAsReadAction extends AbstractRestPostMarkAsReadAction {
@Inject
public RestInvitationCertificationMarkAsReadAction(Settings settings, RestController controller, Client client,
RestSecurityController securityController,
UserInvitationService service) {
super(settings, controller, client, securityController, UserInvitationService.INDEX, UserInvitationService.CERTIFICATION_TYPE,
(id, signature) -> {
service.markInvitationAsRead(id, signature);
});
}
}
\ No newline at end of file
......@@ -115,15 +115,17 @@ public class GroupService extends AbstractService {
public String indexRecordProfileFromJson(String profileJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(profileJson);
String name = getName(actualObj);
String title = getTitle(actualObj);
String id = computeIdFromTitle(title);
String issuer = getIssuer(actualObj);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Indexing a user profile from issuer [%s]", name.substring(0, 8)));
logger.debug(String.format("Indexing group [%s] from issuer [%s]", id, issuer.substring(0, 8)));
}
IndexResponse response = client.prepareIndex(INDEX, RECORD_TYPE)
.setSource(profileJson)
.setId(name) // always use the name as id
.setId(id)
.setRefresh(false)
.execute().actionGet();
return response.getId();
......@@ -133,39 +135,29 @@ public class GroupService extends AbstractService {
* Update a record
* @param recordJson
*/
public ListenableActionFuture<UpdateResponse> updateRecordFromJson(String recordJson, String id) {
public ListenableActionFuture<UpdateResponse> updateRecordFromJson(String id, String recordJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(recordJson);
String name = getName(actualObj);
if (!Objects.equals(name, id)) {
throw new AccessDeniedException(String.format("Could not update this document: not issuer."));
}
if (logger.isDebugEnabled()) {
logger.debug(String.format("Updating a group from name [%s]", name));
logger.debug(String.format("Updating group [%s]", id));
}
return client.prepareUpdate(INDEX, RECORD_TYPE, name)
return client.prepareUpdate(INDEX, RECORD_TYPE, id)
.setDoc(recordJson)
.execute();
}
public String getTitleById(String id) {
protected String getName(JsonNode actualObj) {
return getMandatoryField(actualObj, UserGroup.PROPERTY_NAME).asText();
}
public String getTitleByName(String name) {
Object title = getFieldById(INDEX, RECORD_TYPE, name, UserGroup.PROPERTY_NAME);
Object title = getFieldById(INDEX, RECORD_TYPE, id, UserGroup.PROPERTY_TITLE);
if (title == null) return null;
return title.toString();
}
public Map<String, String> getTitlesByNames(Set<String> names) {
public Map<String, String> getTitlesByNames(Set<String> ids) {
Map<String, Object> titles = getFieldByIds(INDEX, RECORD_TYPE, names, UserGroup.PROPERTY_NAME);
Map<String, Object> titles = getFieldByIds(INDEX, RECORD_TYPE, ids, UserGroup.PROPERTY_TITLE);
if (MapUtils.isEmpty(titles)) return null;
Map<String, String> result = new HashMap<>();
titles.entrySet().stream().forEach((entry) -> result.put(entry.getKey(), entry.getValue().toString()));
......@@ -175,6 +167,29 @@ public class GroupService extends AbstractService {
/* -- Internal methods -- */
protected String getTitle(JsonNode actualObj) {
return getMandatoryField(actualObj, UserGroup.PROPERTY_TITLE).asText();
}
protected String computeIdFromTitle(String title) {
return computeIdFromTitle(title, 0);
}
protected String computeIdFromTitle(String title, int counter) {
String id = title.replaceAll("\\s+", "");
id = id.replaceAll("[^a-zA−Z_-]+", "");
if (counter > 0) {
id += "_" + counter;
}
if (!isDocumentExists(INDEX, RECORD_TYPE, id)) {
return id;
}
return computeIdFromTitle(title, counter+1);
}
public XContentBuilder createRecordType() {
String stringAnalyzer = pluginSettings.getDefaultStringAnalyzer();
......@@ -210,6 +225,18 @@ public class GroupService extends AbstractService {
.field("index", "not_analyzed")
.endObject()
// hash
.startObject("hash")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// signature
.startObject("signature")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// avatar
.startObject("avatar")
.field("type", "attachment")
......
......@@ -124,10 +124,14 @@ public class HistoryService extends AbstractService {
throw new NotFoundException(String.format("Index [%s] not exists.", index));
}
// Special case for message: check if issuer is recipient
// Special case for message: check if deletion issuer is the message recipient
if (MessageService.INDEX.equals(index) && MessageService.INBOX_TYPE.equals(type)) {
checkSameDocumentField(index, type, id, MessageRecord.PROPERTY_RECIPIENT, issuer);
}
// Special case for invitation: check if deletion issuer is the invitation recipient
else if (UserInvitationService.INDEX.equals(index)) {
checkSameDocumentField(index, type, id, MessageRecord.PROPERTY_RECIPIENT, issuer);
}
else {
// Check document issuer
checkSameDocumentIssuer(index, type, id, issuer);
......
......@@ -218,7 +218,13 @@ public class MessageService extends AbstractService {
.field("index", "not_analyzed")
.endObject()
// content
// title (encrypted)
.startObject("title")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// content (encrypted)
.startObject("content")
.field("type", "string")
.field("index", "not_analyzed")
......
......@@ -37,6 +37,8 @@ public class ServiceModule extends AbstractModule implements Module {
bind(UserEventService.class).asEagerSingleton();
bind(UserInvitationService.class).asEagerSingleton();
bind(BlockchainUserEventService.class).asEagerSingleton();
bind(SynchroService.class).asEagerSingleton();
......
......@@ -67,6 +67,7 @@ public class SynchroService extends AbstractSynchroService {
importUserChanges(result, peer, sinceTime);
importMessageChanges(result, peer, sinceTime);
importGroupChanges(result, peer, sinceTime);
importInvitationChanges(result, peer, sinceTime);
long duration = System.currentTimeMillis() - time;
logger.info(String.format("[%s] Synchronizing user data since %s [OK] %s (ins %s ms)", peer.toString(), sinceTime, result.toString(), duration));
......@@ -96,4 +97,8 @@ public class SynchroService extends AbstractSynchroService {
importChanges(result, peer, GroupService.INDEX, GroupService.RECORD_TYPE, sinceTime);
}
protected void importInvitationChanges(SynchroResult result, Peer peer, long sinceTime) {
importChanges(result, peer, UserInvitationService.INDEX, UserInvitationService.CERTIFICATION_TYPE, sinceTime);
}
}
package org.duniter.elasticsearch.user.service;
/*
* #%L
* UCoin Java Client :: Core API
* %%
* Copyright (C) 2014 - 2015 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import org.duniter.core.client.model.ModelUtils;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.service.CryptoService;
import org.duniter.elasticsearch.exception.InvalidSignatureException;
import org.duniter.elasticsearch.user.PluginSettings;
import org.duniter.elasticsearch.user.model.Message;
import org.duniter.elasticsearch.user.model.UserEvent;
import org.duniter.elasticsearch.user.model.UserEventCodes;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.nuiton.i18n.I18n;
import java.io.IOException;
import java.util.Map;
/**
* Created by Benoit on 30/03/2015.
*/
public class UserInvitationService extends AbstractService {
public static final String INDEX = "invitation";
public static final String CERTIFICATION_TYPE = "certification";
private final UserEventService userEventService;
@Inject
public UserInvitationService(Client client, PluginSettings settings,
CryptoService cryptoService, UserEventService userEventService) {
super("duniter." + INDEX, client, settings, cryptoService);
this.userEventService = userEventService;
}
/**
* Delete blockchain index, and all data
* @throws JsonProcessingException
*/
public UserInvitationService deleteIndex() {
deleteIndexIfExists(INDEX);
return this;
}
public boolean existsIndex() {
return super.existsIndex(INDEX);
}
/**
* Create index need for blockchain registry, if need
*/
public UserInvitationService createIndexIfNotExists() {
try {
if (!existsIndex(INDEX)) {
createIndex();
}
}
catch(JsonProcessingException e) {
throw new TechnicalException(String.format("Error while creating index [%s]", INDEX));
}
return this;
}
/**
* Create index need for category registry
* @throws JsonProcessingException
*/
public UserInvitationService createIndex() throws JsonProcessingException {
logger.info(String.format("Creating index [%s/%s]", INDEX, CERTIFICATION_TYPE));
CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(INDEX);
Settings indexSettings = Settings.settingsBuilder()
.put("number_of_shards", 2)
.put("number_of_replicas", 1)
.build();
createIndexRequestBuilder.setSettings(indexSettings);
createIndexRequestBuilder.addMapping(CERTIFICATION_TYPE, createCertificationType());
createIndexRequestBuilder.execute().actionGet();
return this;
}
public String indexCertificationInvitationFromJson(String recordJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(recordJson);
String issuer = getIssuer(actualObj);
String recipient = getMandatoryField(actualObj, Message.PROPERTY_RECIPIENT).asText();
Long time = getMandatoryField(actualObj, Message.PROPERTY_TIME).asLong();
if (logger.isDebugEnabled()) {
logger.debug(String.format("Indexing a invitation to certify from issuer [%s]", issuer.substring(0, 8)));
}
IndexResponse response = client.prepareIndex(INDEX, CERTIFICATION_TYPE)
.setSource(recordJson)
.setRefresh(false)
.execute().actionGet();
String invitationId = response.getId();
// Notify recipient
userEventService.notifyUser(UserEvent.newBuilder(UserEvent.EventType.INFO, UserEventCodes.INVITATION_TO_CERTIFY.name())
.setRecipient(recipient)
.setMessage(I18n.n("duniter.invitation.cert.received"), issuer, ModelUtils.minifyPubkey(issuer))
.setTime(time)
.setReference(INDEX, CERTIFICATION_TYPE, invitationId)
.build());
return invitationId;
}
public void markInvitationAsRead(String id, String signature) {
Map<String, Object> fields = getMandatoryFieldsById(INDEX, CERTIFICATION_TYPE, id, Message.PROPERTY_HASH, Message.PROPERTY_RECIPIENT);
String recipient = fields.get(UserEvent.PROPERTY_RECIPIENT).toString();
String hash = fields.get(UserEvent.PROPERTY_HASH).toString();
// Check signature
boolean valid = cryptoService.verify(hash, signature, recipient);
if (!valid) {
throw new InvalidSignatureException("Invalid signature: only the recipient can mark an message as read.");
}
UpdateRequestBuilder request = client.prepareUpdate(INDEX, CERTIFICATION_TYPE, id)
.setDoc("read_signature", signature);
request.execute();
}
/* -- Internal methods -- */
public XContentBuilder createCertificationType() {
return createMapping(CERTIFICATION_TYPE);
}
public XContentBuilder createMapping(String typeName) {
try {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject(typeName)
.startObject("properties")
// issuer
.startObject("issuer")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// recipient
.startObject("recipient")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// time
.startObject("time")
.field("type", "integer")
.endObject()
// nonce
.startObject("nonce")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// content (encrypted)
.startObject("content")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// read_signature
.startObject("read_signature")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject().endObject();
return mapping;
}
catch(IOException ioe) {
throw new TechnicalException(String.format("Error while getting mapping for index [%s/%s]: %s", INDEX, CERTIFICATION_TYPE, ioe.getMessage()), ioe);
}
}
}
......@@ -88,20 +88,34 @@ curl -XPOST "http://127.0.0.1:9200/user/event/_search?pretty" -d'
echo "--- GET market pictures content_type--- "
curl -XPOST "http://127.0.0.1:9200/market/record/_search?pretty" -d'
curl -XPOST "http://data.gtest.duniter.fr/user/profile/_search?pretty" -d'
{
query: {
constant_score: {
filter: [
{term: { issuer: "5ocqzyDMMWf1V8bsoNhWb1iNwax1e9M7VTUN6navs8of"}}
{terms: { issuer: ["5ocqzyDMMWf1V8bsoNhWb1iNwax1e9M7VTUN6navs8of"]}}
]
}
},
sort : [
{ "time" : {"order" : "desc"}}
],
from: 0,
size: 3,
_source: ["avatar._content_type"]
size: 100,
_source: ["title", "avatar._content_type"]
}'
echo "--- GET user event count --- "
curl -XPOST "http://data.gtest.duniter.fr/user/event/_search?pretty" -d'
{
from: 0,
size: 0,
_source: false
}'
echo "--- GET message count --- "
curl -XPOST "http://data.gtest.duniter.fr/message/record/_search?pretty" -d'
{
from: 0,
size: 10,
_source: ["nonce"]
}'
#!/bin/sh
echo "--- COUNT query --- "
curl -XPOST "http://127.0.0.1:9200/_search?pretty" -d'
{
query: {
indices : {
"indices" : ["user", "registry", "currency"],
"query" : { "term" : { "tags" : "gtest" } },
"no_match_query" : { "term" : { "tags" : "gtest" } }
}
},
from: 0,
size: 100
}'
duniter.event.NODE_BMA_DOWN=Duniter node [%1$s\:%2$s] is DOWN\: no access from ES node [%3$s]. Last connexion at %4$d. Blockchain indexation waiting.
duniter.event.NODE_BMA_UP=Duniter node [%1$s\:%2$s] is UP again.
duniter.event.NODE_STARTED=Node started on cluster Duniter4j ES [%s]
duniter.invitation.cert.received=
duniter.user.event.active=
duniter.user.event.cert.received=
duniter.user.event.cert.sent=
......
duniter.event.NODE_BMA_DOWN=Noeud Duniter [%1$s\:%2$s] non joignable, depuis le noeud ES API [%3$s]. Dernière connexion à %4$d. Indexation de blockchain en attente.
duniter.event.NODE_BMA_UP=Noeud Duniter [%1$s\:%2$s] à nouveau accessible.
duniter.event.NODE_STARTED=Noeud ES API démarré sur le cluster Duniter [%1$s]
duniter.invitation.cert.received=%2$s vous invite à certifier une identité.
duniter.user.event.cert.received=%2$s vous a certifié (certification prise en compte).
duniter.user.event.cert.sent=Votre certification de %2$s a été pris en compte.
duniter.user.event.message.received=Vous avez reçu un message de %2$s
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment