Skip to content
Snippets Groups Projects
Commit 99c72b49 authored by Benoit Lavenier's avatar Benoit Lavenier
Browse files

- add index for invitations

- update group index: remove group name (use attribute 'id' as ES id)
parent 88587f8c
Branches
Tags
No related merge requests found
Showing
with 429 additions and 47 deletions
......@@ -27,24 +27,14 @@ package org.duniter.core.client.model.elasticsearch;
*/
public class UserGroup extends Record {
public static final String PROPERTY_NAME="name";
public static final String PROPERTY_TITLE="title";
public static final String PROPERTY_DESCRIPTION="description";
public static final String PROPERTY_CREATION_TIME="creationTime";
private String name;
private String title;
private String description;
private Long creationTime;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getTitle() {
return title;
}
......
......@@ -138,8 +138,8 @@ duniter.keyring.password: def
# Enable security, to disable HTTP access to the default ES admin API
#
#duniter.security.enable: false
duniter.security.enable: true
duniter.security.enable: false
#duniter.security.enable: true
#
# Security token prefix (default: 'duniter-')
#
......@@ -153,9 +153,9 @@ duniter.security.enable: true
#
# Should synchronize data using P2P
#
duniter.data.sync.enable: false
#duniter.data.sync.host: data.duniter.fr
#duniter.data.sync.port: 80
duniter.data.sync.enable: true
duniter.data.sync.host: data.gtest.duniter.fr
duniter.data.sync.port: 80
# ---------------------------------- Duniter4j SMTP server -------------------------
#
......
......@@ -237,7 +237,6 @@ public abstract class AbstractService implements Bean {
}
}
protected String getIssuer(JsonNode actualObj) {
return getMandatoryField(actualObj, Record.PROPERTY_ISSUER).asText();
}
......@@ -259,7 +258,7 @@ public abstract class AbstractService implements Bean {
Map<String, Object> fields = getFieldsById(index, type, docId, fieldNames);
if (MapUtils.isEmpty(fields)) throw new NotFoundException(String.format("Document [%s/%s/%s] not exists.", index, type, docId));
Arrays.stream(fieldNames).forEach((fieldName) -> {
if (!fields.containsKey(fieldName)) throw new NotFoundException(String.format("Document [%s/%s/%s] should have the madatory field [%s].", index, type, docId, fieldName));
if (!fields.containsKey(fieldName)) throw new NotFoundException(String.format("Document [%s/%s/%s] should have the mandatory field [%s].", index, type, docId, fieldName));
});
return fields;
}
......
......@@ -172,6 +172,7 @@ public abstract class AbstractSynchroService extends AbstractService {
JsonNode node;
try {
HttpPost httpPost = new HttpPost(httpService.getPath(peer, fromIndex, fromType, "_search"));
httpPost.setHeader("Content-Type", "application/json;charset=UTF-8");
httpPost.setEntity(new ByteArrayEntity(bos.bytes().array()));
if (logger.isDebugEnabled()) {
logger.debug(String.format("[%s] [%s/%s] Sending POST request: %s", peer, fromIndex, fromType, new String(bos.bytes().array())));
......
......@@ -106,6 +106,9 @@ public class PluginInit extends AbstractLifecycleComponent<PluginInit> {
injector.getInstance(GroupService.class)
.deleteIndex()
.createIndexIfNotExists();
injector.getInstance(UserInvitationService.class)
.deleteIndex()
.createIndexIfNotExists();
if (logger.isInfoEnabled()) {
logger.info("Reloading all Duniter indices... [OK]");
......@@ -119,6 +122,7 @@ public class PluginInit extends AbstractLifecycleComponent<PluginInit> {
injector.getInstance(UserService.class).createIndexIfNotExists();
injector.getInstance(MessageService.class).createIndexIfNotExists();
injector.getInstance(GroupService.class).createIndexIfNotExists();
injector.getInstance(UserInvitationService.class).createIndexIfNotExists();
if (logger.isInfoEnabled()) {
logger.info("Checking Duniter indices... [OK]");
......
......@@ -45,6 +45,8 @@ public enum UserEventCodes {
CERT_RECEIVED,
// Message
MESSAGE_RECEIVED
MESSAGE_RECEIVED,
// Invitation
INVITATION_TO_CERTIFY
}
......@@ -25,6 +25,8 @@ package org.duniter.elasticsearch.user.rest;
import org.duniter.elasticsearch.user.rest.group.RestGroupIndexAction;
import org.duniter.elasticsearch.user.rest.group.RestGroupUpdateAction;
import org.duniter.elasticsearch.user.rest.history.RestHistoryDeleteIndexAction;
import org.duniter.elasticsearch.user.rest.invitation.RestInvitationCertificationIndexAction;
import org.duniter.elasticsearch.user.rest.invitation.RestInvitationCertificationMarkAsReadAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageInboxIndexAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageInboxMarkAsReadAction;
import org.duniter.elasticsearch.user.rest.message.RestMessageOutboxIndexAction;
......@@ -61,6 +63,10 @@ public class RestModule extends AbstractModule implements Module {
bind(RestMessageOutboxIndexAction.class).asEagerSingleton();
bind(RestMessageInboxMarkAsReadAction.class).asEagerSingleton();
// Invitation
bind(RestInvitationCertificationIndexAction.class).asEagerSingleton();
bind(RestInvitationCertificationMarkAsReadAction.class).asEagerSingleton();
// Backward compatibility
{
// message/record
......
package org.duniter.elasticsearch.user.rest.invitation;
/*
* #%L
* duniter4j-elasticsearch-plugin
* %%
* Copyright (C) 2014 - 2016 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import org.duniter.elasticsearch.rest.AbstractRestPostIndexAction;
import org.duniter.elasticsearch.rest.security.RestSecurityController;
import org.duniter.elasticsearch.user.service.UserInvitationService;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestController;
public class RestInvitationCertificationIndexAction extends AbstractRestPostIndexAction {
@Inject
public RestInvitationCertificationIndexAction(Settings settings, RestController controller, Client client,
RestSecurityController securityController,
final UserInvitationService service) {
super(settings, controller, client, securityController,
UserInvitationService.INDEX,
UserInvitationService.CERTIFICATION_TYPE,
json -> service.indexCertificationInvitationFromJson(json));
}
}
\ No newline at end of file
package org.duniter.elasticsearch.user.rest.invitation;
/*
* #%L
* duniter4j-elasticsearch-plugin
* %%
* Copyright (C) 2014 - 2016 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import org.duniter.elasticsearch.rest.AbstractRestPostMarkAsReadAction;
import org.duniter.elasticsearch.rest.security.RestSecurityController;
import org.duniter.elasticsearch.user.service.UserInvitationService;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.rest.RestController;
public class RestInvitationCertificationMarkAsReadAction extends AbstractRestPostMarkAsReadAction {
@Inject
public RestInvitationCertificationMarkAsReadAction(Settings settings, RestController controller, Client client,
RestSecurityController securityController,
UserInvitationService service) {
super(settings, controller, client, securityController, UserInvitationService.INDEX, UserInvitationService.CERTIFICATION_TYPE,
(id, signature) -> {
service.markInvitationAsRead(id, signature);
});
}
}
\ No newline at end of file
......@@ -115,15 +115,17 @@ public class GroupService extends AbstractService {
public String indexRecordProfileFromJson(String profileJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(profileJson);
String name = getName(actualObj);
String title = getTitle(actualObj);
String id = computeIdFromTitle(title);
String issuer = getIssuer(actualObj);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Indexing a user profile from issuer [%s]", name.substring(0, 8)));
logger.debug(String.format("Indexing group [%s] from issuer [%s]", id, issuer.substring(0, 8)));
}
IndexResponse response = client.prepareIndex(INDEX, RECORD_TYPE)
.setSource(profileJson)
.setId(name) // always use the name as id
.setId(id)
.setRefresh(false)
.execute().actionGet();
return response.getId();
......@@ -133,39 +135,29 @@ public class GroupService extends AbstractService {
* Update a record
* @param recordJson
*/
public ListenableActionFuture<UpdateResponse> updateRecordFromJson(String recordJson, String id) {
public ListenableActionFuture<UpdateResponse> updateRecordFromJson(String id, String recordJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(recordJson);
String name = getName(actualObj);
if (!Objects.equals(name, id)) {
throw new AccessDeniedException(String.format("Could not update this document: not issuer."));
}
if (logger.isDebugEnabled()) {
logger.debug(String.format("Updating a group from name [%s]", name));
logger.debug(String.format("Updating group [%s]", id));
}
return client.prepareUpdate(INDEX, RECORD_TYPE, name)
return client.prepareUpdate(INDEX, RECORD_TYPE, id)
.setDoc(recordJson)
.execute();
}
public String getTitleById(String id) {
protected String getName(JsonNode actualObj) {
return getMandatoryField(actualObj, UserGroup.PROPERTY_NAME).asText();
}
public String getTitleByName(String name) {
Object title = getFieldById(INDEX, RECORD_TYPE, name, UserGroup.PROPERTY_NAME);
Object title = getFieldById(INDEX, RECORD_TYPE, id, UserGroup.PROPERTY_TITLE);
if (title == null) return null;
return title.toString();
}
public Map<String, String> getTitlesByNames(Set<String> names) {
public Map<String, String> getTitlesByNames(Set<String> ids) {
Map<String, Object> titles = getFieldByIds(INDEX, RECORD_TYPE, names, UserGroup.PROPERTY_NAME);
Map<String, Object> titles = getFieldByIds(INDEX, RECORD_TYPE, ids, UserGroup.PROPERTY_TITLE);
if (MapUtils.isEmpty(titles)) return null;
Map<String, String> result = new HashMap<>();
titles.entrySet().stream().forEach((entry) -> result.put(entry.getKey(), entry.getValue().toString()));
......@@ -175,6 +167,29 @@ public class GroupService extends AbstractService {
/* -- Internal methods -- */
protected String getTitle(JsonNode actualObj) {
return getMandatoryField(actualObj, UserGroup.PROPERTY_TITLE).asText();
}
protected String computeIdFromTitle(String title) {
return computeIdFromTitle(title, 0);
}
protected String computeIdFromTitle(String title, int counter) {
String id = title.replaceAll("\\s+", "");
id = id.replaceAll("[^a-zA−Z_-]+", "");
if (counter > 0) {
id += "_" + counter;
}
if (!isDocumentExists(INDEX, RECORD_TYPE, id)) {
return id;
}
return computeIdFromTitle(title, counter+1);
}
public XContentBuilder createRecordType() {
String stringAnalyzer = pluginSettings.getDefaultStringAnalyzer();
......@@ -210,6 +225,18 @@ public class GroupService extends AbstractService {
.field("index", "not_analyzed")
.endObject()
// hash
.startObject("hash")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// signature
.startObject("signature")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// avatar
.startObject("avatar")
.field("type", "attachment")
......
......@@ -124,10 +124,14 @@ public class HistoryService extends AbstractService {
throw new NotFoundException(String.format("Index [%s] not exists.", index));
}
// Special case for message: check if issuer is recipient
// Special case for message: check if deletion issuer is the message recipient
if (MessageService.INDEX.equals(index) && MessageService.INBOX_TYPE.equals(type)) {
checkSameDocumentField(index, type, id, MessageRecord.PROPERTY_RECIPIENT, issuer);
}
// Special case for invitation: check if deletion issuer is the invitation recipient
else if (UserInvitationService.INDEX.equals(index)) {
checkSameDocumentField(index, type, id, MessageRecord.PROPERTY_RECIPIENT, issuer);
}
else {
// Check document issuer
checkSameDocumentIssuer(index, type, id, issuer);
......
......@@ -218,7 +218,13 @@ public class MessageService extends AbstractService {
.field("index", "not_analyzed")
.endObject()
// content
// title (encrypted)
.startObject("title")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// content (encrypted)
.startObject("content")
.field("type", "string")
.field("index", "not_analyzed")
......
......@@ -37,6 +37,8 @@ public class ServiceModule extends AbstractModule implements Module {
bind(UserEventService.class).asEagerSingleton();
bind(UserInvitationService.class).asEagerSingleton();
bind(BlockchainUserEventService.class).asEagerSingleton();
bind(SynchroService.class).asEagerSingleton();
......
......@@ -67,6 +67,7 @@ public class SynchroService extends AbstractSynchroService {
importUserChanges(result, peer, sinceTime);
importMessageChanges(result, peer, sinceTime);
importGroupChanges(result, peer, sinceTime);
importInvitationChanges(result, peer, sinceTime);
long duration = System.currentTimeMillis() - time;
logger.info(String.format("[%s] Synchronizing user data since %s [OK] %s (ins %s ms)", peer.toString(), sinceTime, result.toString(), duration));
......@@ -96,4 +97,8 @@ public class SynchroService extends AbstractSynchroService {
importChanges(result, peer, GroupService.INDEX, GroupService.RECORD_TYPE, sinceTime);
}
protected void importInvitationChanges(SynchroResult result, Peer peer, long sinceTime) {
importChanges(result, peer, UserInvitationService.INDEX, UserInvitationService.CERTIFICATION_TYPE, sinceTime);
}
}
package org.duniter.elasticsearch.user.service;
/*
* #%L
* UCoin Java Client :: Core API
* %%
* Copyright (C) 2014 - 2015 EIS
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program. If not, see
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import org.duniter.core.client.model.ModelUtils;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.service.CryptoService;
import org.duniter.elasticsearch.exception.InvalidSignatureException;
import org.duniter.elasticsearch.user.PluginSettings;
import org.duniter.elasticsearch.user.model.Message;
import org.duniter.elasticsearch.user.model.UserEvent;
import org.duniter.elasticsearch.user.model.UserEventCodes;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.update.UpdateRequestBuilder;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.nuiton.i18n.I18n;
import java.io.IOException;
import java.util.Map;
/**
* Created by Benoit on 30/03/2015.
*/
public class UserInvitationService extends AbstractService {
public static final String INDEX = "invitation";
public static final String CERTIFICATION_TYPE = "certification";
private final UserEventService userEventService;
@Inject
public UserInvitationService(Client client, PluginSettings settings,
CryptoService cryptoService, UserEventService userEventService) {
super("duniter." + INDEX, client, settings, cryptoService);
this.userEventService = userEventService;
}
/**
* Delete blockchain index, and all data
* @throws JsonProcessingException
*/
public UserInvitationService deleteIndex() {
deleteIndexIfExists(INDEX);
return this;
}
public boolean existsIndex() {
return super.existsIndex(INDEX);
}
/**
* Create index need for blockchain registry, if need
*/
public UserInvitationService createIndexIfNotExists() {
try {
if (!existsIndex(INDEX)) {
createIndex();
}
}
catch(JsonProcessingException e) {
throw new TechnicalException(String.format("Error while creating index [%s]", INDEX));
}
return this;
}
/**
* Create index need for category registry
* @throws JsonProcessingException
*/
public UserInvitationService createIndex() throws JsonProcessingException {
logger.info(String.format("Creating index [%s/%s]", INDEX, CERTIFICATION_TYPE));
CreateIndexRequestBuilder createIndexRequestBuilder = client.admin().indices().prepareCreate(INDEX);
Settings indexSettings = Settings.settingsBuilder()
.put("number_of_shards", 2)
.put("number_of_replicas", 1)
.build();
createIndexRequestBuilder.setSettings(indexSettings);
createIndexRequestBuilder.addMapping(CERTIFICATION_TYPE, createCertificationType());
createIndexRequestBuilder.execute().actionGet();
return this;
}
public String indexCertificationInvitationFromJson(String recordJson) {
JsonNode actualObj = readAndVerifyIssuerSignature(recordJson);
String issuer = getIssuer(actualObj);
String recipient = getMandatoryField(actualObj, Message.PROPERTY_RECIPIENT).asText();
Long time = getMandatoryField(actualObj, Message.PROPERTY_TIME).asLong();
if (logger.isDebugEnabled()) {
logger.debug(String.format("Indexing a invitation to certify from issuer [%s]", issuer.substring(0, 8)));
}
IndexResponse response = client.prepareIndex(INDEX, CERTIFICATION_TYPE)
.setSource(recordJson)
.setRefresh(false)
.execute().actionGet();
String invitationId = response.getId();
// Notify recipient
userEventService.notifyUser(UserEvent.newBuilder(UserEvent.EventType.INFO, UserEventCodes.INVITATION_TO_CERTIFY.name())
.setRecipient(recipient)
.setMessage(I18n.n("duniter.invitation.cert.received"), issuer, ModelUtils.minifyPubkey(issuer))
.setTime(time)
.setReference(INDEX, CERTIFICATION_TYPE, invitationId)
.build());
return invitationId;
}
public void markInvitationAsRead(String id, String signature) {
Map<String, Object> fields = getMandatoryFieldsById(INDEX, CERTIFICATION_TYPE, id, Message.PROPERTY_HASH, Message.PROPERTY_RECIPIENT);
String recipient = fields.get(UserEvent.PROPERTY_RECIPIENT).toString();
String hash = fields.get(UserEvent.PROPERTY_HASH).toString();
// Check signature
boolean valid = cryptoService.verify(hash, signature, recipient);
if (!valid) {
throw new InvalidSignatureException("Invalid signature: only the recipient can mark an message as read.");
}
UpdateRequestBuilder request = client.prepareUpdate(INDEX, CERTIFICATION_TYPE, id)
.setDoc("read_signature", signature);
request.execute();
}
/* -- Internal methods -- */
public XContentBuilder createCertificationType() {
return createMapping(CERTIFICATION_TYPE);
}
public XContentBuilder createMapping(String typeName) {
try {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject(typeName)
.startObject("properties")
// issuer
.startObject("issuer")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// recipient
.startObject("recipient")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// time
.startObject("time")
.field("type", "integer")
.endObject()
// nonce
.startObject("nonce")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// content (encrypted)
.startObject("content")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
// read_signature
.startObject("read_signature")
.field("type", "string")
.field("index", "not_analyzed")
.endObject()
.endObject()
.endObject().endObject();
return mapping;
}
catch(IOException ioe) {
throw new TechnicalException(String.format("Error while getting mapping for index [%s/%s]: %s", INDEX, CERTIFICATION_TYPE, ioe.getMessage()), ioe);
}
}
}
......@@ -88,20 +88,34 @@ curl -XPOST "http://127.0.0.1:9200/user/event/_search?pretty" -d'
echo "--- GET market pictures content_type--- "
curl -XPOST "http://127.0.0.1:9200/market/record/_search?pretty" -d'
curl -XPOST "http://data.gtest.duniter.fr/user/profile/_search?pretty" -d'
{
query: {
constant_score: {
filter: [
{term: { issuer: "5ocqzyDMMWf1V8bsoNhWb1iNwax1e9M7VTUN6navs8of"}}
{terms: { issuer: ["5ocqzyDMMWf1V8bsoNhWb1iNwax1e9M7VTUN6navs8of"]}}
]
}
},
sort : [
{ "time" : {"order" : "desc"}}
],
from: 0,
size: 3,
_source: ["avatar._content_type"]
size: 100,
_source: ["title", "avatar._content_type"]
}'
echo "--- GET user event count --- "
curl -XPOST "http://data.gtest.duniter.fr/user/event/_search?pretty" -d'
{
from: 0,
size: 0,
_source: false
}'
echo "--- GET message count --- "
curl -XPOST "http://data.gtest.duniter.fr/message/record/_search?pretty" -d'
{
from: 0,
size: 10,
_source: ["nonce"]
}'
#!/bin/sh
echo "--- COUNT query --- "
curl -XPOST "http://127.0.0.1:9200/_search?pretty" -d'
{
query: {
indices : {
"indices" : ["user", "registry", "currency"],
"query" : { "term" : { "tags" : "gtest" } },
"no_match_query" : { "term" : { "tags" : "gtest" } }
}
},
from: 0,
size: 100
}'
duniter.event.NODE_BMA_DOWN=Duniter node [%1$s\:%2$s] is DOWN\: no access from ES node [%3$s]. Last connexion at %4$d. Blockchain indexation waiting.
duniter.event.NODE_BMA_UP=Duniter node [%1$s\:%2$s] is UP again.
duniter.event.NODE_STARTED=Node started on cluster Duniter4j ES [%s]
duniter.invitation.cert.received=
duniter.user.event.active=
duniter.user.event.cert.received=
duniter.user.event.cert.sent=
......
duniter.event.NODE_BMA_DOWN=Noeud Duniter [%1$s\:%2$s] non joignable, depuis le noeud ES API [%3$s]. Dernière connexion à %4$d. Indexation de blockchain en attente.
duniter.event.NODE_BMA_UP=Noeud Duniter [%1$s\:%2$s] à nouveau accessible.
duniter.event.NODE_STARTED=Noeud ES API démarré sur le cluster Duniter [%1$s]
duniter.invitation.cert.received=%2$s vous invite à certifier une identité.
duniter.user.event.cert.received=%2$s vous a certifié (certification prise en compte).
duniter.user.event.cert.sent=Votre certification de %2$s a été pris en compte.
duniter.user.event.message.received=Vous avez reçu un message de %2$s
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment