Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • clients/cesium-grp/cesium-plus-pod
  • clients/java/duniter4j
  • ji_emme/duniter4j
  • dvermd/cesium-plus-pod
  • okayotanoka/cesium-plus-pod
  • pokapow/cesium-plus-pod
  • pini-gh/cesium-plus-pod
7 results
Show changes
Showing
with 2719 additions and 31 deletions
package org.duniter.elasticsearch.client.model.query;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonInclude;
import lombok.Builder;
import lombok.Data;
import java.util.Map;
@Data
@Builder
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SearchQuery {
private GeoBoundingBoxQuery geoBoundingBox;
private QueryString queryString;
private ExistsQuery exists;
private BoolQuery bool;
private ConstantScoreQuery constantScore;
private Map<String, String> match;
private Map<String, String> matchPhrase;
private Map<String, String> matchPhrasePrefix;
private Map<String, String> prefix;
private Map<String, String> term;
private Map<String, String[]> terms;
private Map<String, RangePartQuery> range;
@JsonGetter("constant_score")
public ConstantScoreQuery getConstantScore() {
return constantScore;
}
@JsonGetter("query_string")
public QueryString getQueryString() {
return queryString;
}
@JsonGetter("geo_bounding_box")
public GeoBoundingBoxQuery getGeoBoundingBox() {
return geoBoundingBox;
}
@JsonGetter("match_phrase")
public Map<String, String> getMatchPhrase() {
return matchPhrase;
}
@JsonGetter("match_phrase_prefix")
public Map<String, String> getMatchPhrasePrefix() {
return matchPhrasePrefix;
}
}
package org.duniter.elasticsearch.client.model.query;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.annotation.JsonGetter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.common.collect.ImmutableMap;
import lombok.Builder;
import lombok.Data;
import org.duniter.core.util.StringUtils;
import org.duniter.elasticsearch.model.SortDirection;
import java.util.Map;
@Data
@Builder
@JsonInclude(JsonInclude.Include.NON_NULL)
public class SearchRequest {
public static class SearchRequestBuilder{
public SearchRequestBuilder queryString(String queryString) {
this.query(SearchQuery.builder()
.queryString(QueryString.builder()
.query(queryString)
.build())
.build());
return SearchRequestBuilder.this;
}
public SearchRequestBuilder sortBy(String field, SortDirection direction) {
if (StringUtils.isNotBlank(field)) {
this.sort(ImmutableMap.of(field, direction == null || direction == SortDirection.ASC ? "asc" : "desc"));
}
return SearchRequestBuilder.this;
}
}
private SearchQuery query;
Integer from;
Integer size;
String[] source;
Map<String, String> sort;
@JsonGetter("_source")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public String[] getSource() {
return source;
}
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.core.client.service.bma.BaseRemoteServiceImpl;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.service.CryptoService;
import org.duniter.core.util.json.JsonAttributeParser;
import org.duniter.elasticsearch.model.Record;
import org.duniter.elasticsearch.model.Records;
import org.duniter.elasticsearch.model.user.UserEvent;
import org.duniter.elasticsearch.model.user.UserProfile;
import java.io.IOException;
@Slf4j
public abstract class AbstractServiceImpl extends BaseRemoteServiceImpl {
protected static JsonAttributeParser<String> PARSER_HASH = new JsonAttributeParser<>(Record.Fields.HASH, String.class);
protected static JsonAttributeParser<String> PARSER_SIGNATURE = new JsonAttributeParser<>(Record.Fields.SIGNATURE, String.class);
protected Configuration config;
protected CryptoService cryptoService;
protected ObjectMapper objectMapper;
@Override
public void afterPropertiesSet() {
super.afterPropertiesSet();
config = Configuration.instance();
cryptoService = ServiceLocator.instance().getCryptoService();
}
@Override
public void close() throws IOException {
super.close();
}
public <T extends Record> T sendRecord(Peer peer, String path, Wallet wallet, T record) {
ObjectMapper objectMapper = getObjectMapper();
fillDefaults(wallet, record);
// Add hash + signature
T signedRecord = addHashAndSignature(wallet, record);
try {
if (log.isDebugEnabled()) {
String json = objectMapper.writeValueAsString(signedRecord);
log.debug("Sending POST request to [{}]: {}", path, json);
}
HttpPost httpPost = new HttpPost(httpService.getPath(peer, path));
HttpEntity entity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_JSON)
.setBinary(objectMapper.writeValueAsBytes(signedRecord))
.build();
httpPost.setEntity(entity);
// Send it to pod
String id = this.httpService.executeRequest(httpPost, String.class);
// Save the id
signedRecord.setId(id);
return signedRecord;
} catch(JsonProcessingException e) {
throw new TechnicalException("Unable to serialize UserEvent object", e);
}
}
protected <T extends Record> T addHashAndSignature(@NonNull Wallet wallet, @NonNull T record) {
String hash = cryptoService.hash(toJson(record, true));
record.setHash(hash);
String signature = cryptoService.sign(hash, wallet.getSecKey());
record.setSignature(signature);
return record;
}
private <T extends Record> String toJson(T record, boolean cleanHashAndSignature) {
try {
String json = getObjectMapper().writeValueAsString(record);
if (cleanHashAndSignature) {
json = PARSER_SIGNATURE.removeFromJson(json);
json = PARSER_HASH.removeFromJson(json);
}
return json;
} catch(JsonProcessingException e) {
throw new TechnicalException("Unable to serialize UserEvent object", e);
}
}
protected <R extends Record> void fillDefaults(Wallet wallet, R record) {
record.setIssuer(wallet.getPubKeyHash());
// Set time
if (record.getTime() == null) {
record.setTime(System.currentTimeMillis() / 1000);
}
// Set version
if (record.getVersion() == null) {
record.setVersion(Records.PROTOCOL_VERSION);
}
}
protected ObjectMapper getObjectMapper() {
if (objectMapper == null) {
objectMapper = JacksonUtils.getThreadObjectMapper();
}
return objectMapper;
}
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import org.duniter.core.beans.Bean;
import java.io.Closeable;
import java.io.IOException;
public class ServiceLocator extends org.duniter.core.client.service.ServiceLocator implements Closeable {
/**
* The shared instance of this ServiceLocator.
*/
private static ServiceLocator instance = new ServiceLocator();
/**
* Gets the shared instance of this Class
*
* @return the shared service locator instance.
*/
public static ServiceLocator instance() {
return instance;
}
org.duniter.core.client.service.ServiceLocator delegate;
protected ServiceLocator() {
init();
}
public void init() {
if (delegate == null) {
delegate = org.duniter.core.client.service.ServiceLocator.instance();
delegate.init();
}
}
@Override
public void close() throws IOException {
if (delegate != null)
delegate.close();
}
public UserProfileService getUserProfileService() {
return delegate.getBean(UserProfileService.class);
}
public UserSettingsService getUserSettingsService() {
return delegate.getBean(UserSettingsService.class);
}
public <S extends Bean> S getBean(Class<S> clazz) {
if (delegate == null) init();
return delegate.getBean(clazz);
}
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import org.duniter.core.beans.Service;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.client.model.filter.MovementFilter;
import org.duniter.elasticsearch.client.model.filter.UserProfileFilter;
import org.duniter.elasticsearch.model.blockchain.Movement;
import org.duniter.elasticsearch.model.user.UserProfile;
import javax.annotation.Nullable;
import java.util.stream.Stream;
import org.geojson.FeatureCollection;
public interface UserProfileService extends Service {
Stream<UserProfile> findAllByFilter(Peer peer, UserProfileFilter filter, @Nullable Page page);
UserProfile save(Peer peer, Wallet wallet, UserProfile userProfile);
UserProfile update(Peer peer, Wallet wallet, UserProfile userProfile);
FeatureCollection toGeoJson(Iterable<UserProfile> profiles, String... fields);
Stream<Movement> findMovements(Peer peer, MovementFilter filter, @Nullable Page page);
boolean deleteByPubkey(Peer peer, Wallet wallet, String pubkey);
boolean delete(Peer peer, Wallet wallet, UserProfile userProfile);
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableSortedSet;
import com.google.common.collect.Lists;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ArrayUtils;
import org.duniter.core.util.Beans;
import org.duniter.core.util.Preconditions;
import org.duniter.core.util.StringUtils;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.client.model.filter.MovementFilter;
import org.duniter.elasticsearch.client.model.filter.UserProfileFilter;
import org.duniter.elasticsearch.client.model.query.*;
import org.duniter.elasticsearch.dao.user.IUserProfileRepository;
import org.duniter.elasticsearch.model.DeleteRecord;
import org.duniter.elasticsearch.model.blockchain.Movement;
import org.duniter.elasticsearch.model.query.SearchResponse;
import org.duniter.elasticsearch.model.type.Attachment;
import org.duniter.elasticsearch.model.user.UserProfile;
import org.duniter.elasticsearch.model.user.UserSettings;
import org.duniter.elasticsearch.utils.Dates;
import org.duniter.elasticsearch.utils.Geometries;
import org.geojson.Feature;
import org.geojson.FeatureCollection;
import org.geojson.GeoJsonObject;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
public class UserProfileServiceImpl extends AbstractServiceImpl implements UserProfileService {
protected Configuration config;
public interface Uris {
String SEARCH = "/user/profile/_search";
String POST_INSERT = "/user/profile";
String POST_UPDATE = "/user/profile/%s/_update";
String POST_DELETE = "/history/delete";
String SEARCH_MOVEMENTS = "/%s/movement/_search";
}
@Override
public void afterPropertiesSet() {
super.afterPropertiesSet();
config = Configuration.instance();
}
@Override
public void close() throws IOException {
super.close();
}
@Override
public Stream<UserProfile> findAllByFilter(@NonNull Peer peer,
@NonNull UserProfileFilter filter,
@Nullable Page page) {
page = Page.nullToDefault(page);
ObjectMapper objectMapper = getObjectMapper();
try {
List<SearchQuery> matches = Lists.newArrayList();
List<SearchQuery> filters = Lists.newArrayList();
// Issuers
if (ArrayUtils.isNotEmpty(filter.getIssuers())) {
filters.add(SearchQueries
.terms(UserSettings.Fields.ISSUER, filter.getIssuers())
);
}
// Query String
if (StringUtils.isNotBlank(filter.getQueryString())) {
matches.add(SearchQueries.queryString(filter.getQueryString()));
}
if (filter.getBoundingBox() != null) {
filters.add(SearchQueries.exists(UserProfile.Fields.GEO_POINT));
filters.add(SearchQueries.geoBoundingBox(filter.getBoundingBox()));
}
if (filter.getStartDate() != null && filter.getEndDate() != null) {
matches.add(SearchQueries.range(UserProfile.Fields.TIME, RangePartQuery.builder()
.gte(Dates.toUnixTimestamp(filter.getStartDate()))
.lt(Dates.toUnixTimestamp(filter.getEndDate()))
.build()));
}
else if (filter.getStartDate() != null) {
matches.add(SearchQueries.range(UserProfile.Fields.TIME, RangePartQuery.builder()
.gte(Dates.toUnixTimestamp(filter.getStartDate()))
.build()));
}
else if (filter.getEndDate() != null) {
matches.add(SearchQueries.range(UserProfile.Fields.TIME, RangePartQuery.builder()
.lte(Dates.toUnixTimestamp(filter.getEndDate()))
.build()));
}
SearchQuery query = SearchQuery.builder()
.bool(BoolQuery.builder()
.must(matches.toArray(new SearchQuery[0]))
.filter(filters.toArray(new SearchQuery[0]))
.build())
.build();
SearchRequest request = SearchRequest.builder()
.query(query)
.from(page.getFrom())
.size(page.getSize())
.source(filter.getFields())
.sortBy(page.getSortBy(), page.getSortDirection())
.build();
long now = System.currentTimeMillis();
if (log.isDebugEnabled()) {
log.debug("Searching on user profiles using: {}", objectMapper.writeValueAsString(request));
}
HttpPost httpPost = new HttpPost(httpService.getPath(peer, Uris.SEARCH));
HttpEntity entity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_JSON)
.setBinary(objectMapper.writeValueAsBytes(request))
.build();
httpPost.setEntity(entity);
SearchResponse response = httpService.executeRequest(httpPost, SearchResponse.class);
if (log.isDebugEnabled()) {
log.debug("Searching on user profiles [OK] - {} result(s) found (total: {}) in {}ms",
ArrayUtils.size(response.getHits().getHits()),
response.getHits().getTotal(),
System.currentTimeMillis() - now);
}
return Arrays.stream(response.getHits().getHits())
.map(hit -> {
try {
return objectMapper.treeToValue(hit.getSource(), UserProfile.class);
} catch (JsonProcessingException e) {
log.warn("Error during user profile deserialization");
return null;
}
})
.filter(Objects::nonNull);
} catch (JsonProcessingException e) {
throw new TechnicalException(e);
}
}
@Override
public UserProfile save(Peer peer, Wallet wallet, UserProfile userProfile) {
fillDefaults(wallet, userProfile);
return sendRecord(peer, Uris.POST_INSERT, wallet, userProfile);
}
@Override
public UserProfile update(Peer peer, Wallet wallet, @NonNull UserProfile userProfile) {
fillDefaults(wallet, userProfile);
return sendRecord(peer, String.format(Uris.POST_UPDATE, userProfile.getId()), wallet, userProfile);
}
@Override
public Stream<Movement> findMovements(@NonNull Peer peer, @NonNull MovementFilter filter, @Nullable Page page) {
Preconditions.checkNotNull(peer.getCurrency());
page = Page.nullToDefault(page);
ObjectMapper objectMapper = getObjectMapper();
try {
List<SearchQuery> matches = Lists.newArrayList();
List<SearchQuery> filters = Lists.newArrayList();
// Issuers
if (ArrayUtils.isNotEmpty(filter.getIssuers())) {
filters.add(SearchQueries.terms(Movement.Fields.ISSUER, filter.getIssuers()));
}
// Recipients
if (ArrayUtils.isNotEmpty(filter.getRecipients())) {
filters.add(SearchQueries.terms(Movement.Fields.RECIPIENT, filter.getRecipients()));
}
// Pubkey (=issuer or recipient)
if (StringUtils.isNotBlank(filter.getPubkey())) {
matches.add(SearchQueries.queryString(String.format("issuer:%s OR recipient:%s", filter.getPubkey(), filter.getPubkey())));
}
if (filter.getStartDate() != null && filter.getEndDate() != null) {
matches.add(SearchQueries.range(Movement.Fields.MEDIAN_TIME, RangePartQuery.builder()
.gte(Dates.toUnixTimestamp(filter.getStartDate()))
.lt(Dates.toUnixTimestamp(filter.getEndDate()))
.build()));
}
else if (filter.getStartDate() != null) {
matches.add(SearchQueries.range(Movement.Fields.MEDIAN_TIME, RangePartQuery.builder()
.gte(Dates.toUnixTimestamp(filter.getStartDate()))
.build()));
}
else if (filter.getEndDate() != null) {
matches.add(SearchQueries.range(Movement.Fields.MEDIAN_TIME, RangePartQuery.builder()
.lte(Dates.toUnixTimestamp(filter.getEndDate()))
.build()));
}
SearchQuery query = SearchQuery.builder()
.bool(BoolQuery.builder()
.must(matches.toArray(new SearchQuery[0]))
.filter(filters.toArray(new SearchQuery[0]))
.build())
.build();
SearchRequest request = SearchRequest.builder()
.query(query)
.from(page.getFrom())
.size(page.getSize())
.source(filter.getFields())
.build();
long now = System.currentTimeMillis();
if (log.isDebugEnabled()) {
log.debug("Searching on movements using: {}", objectMapper.writeValueAsString(request));
}
HttpPost httpPost = new HttpPost(httpService.getPath(peer, String.format(Uris.SEARCH_MOVEMENTS, peer.getCurrency())));
HttpEntity entity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_JSON)
.setBinary(objectMapper.writeValueAsBytes(request))
.build();
httpPost.setEntity(entity);
SearchResponse response = httpService.executeRequest(httpPost, SearchResponse.class);
if (log.isDebugEnabled()) {
log.debug("Searching on movements [OK] - {} result(s) found in {}ms",
ArrayUtils.size(response.getHits().getHits()),
System.currentTimeMillis() - now);
}
return Arrays.stream(response.getHits().getHits())
.map(hit -> {
try {
return objectMapper.treeToValue(hit.getSource(), Movement.class);
} catch (JsonProcessingException e) {
log.warn("Error during movement deserialization");
return null;
}
})
.filter(Objects::nonNull);
} catch (JsonProcessingException e) {
throw new TechnicalException(e);
}
}
@Override
public boolean delete(@NonNull Peer peer, @NonNull Wallet wallet, @NonNull UserProfile userProfile) {
Preconditions.checkNotNull(userProfile.getIssuer());
return deleteByPubkey(peer, wallet, userProfile.getIssuer());
}
@Override
public boolean deleteByPubkey(Peer peer, Wallet wallet, String pubkey) {
DeleteRecord record = new DeleteRecord();
record.setIndex(IUserProfileRepository.INDEX);
record.setType(IUserProfileRepository.TYPE);
record.setId(pubkey);
DeleteRecord savedRecord = sendRecord(peer, Uris.POST_DELETE, wallet, record);
return StringUtils.isNotBlank(savedRecord.getId());
}
@Override
public FeatureCollection toGeoJson(Iterable<UserProfile> profiles, String... fields) {
FeatureCollection features = new FeatureCollection();
Set<String> fieldList = ArrayUtils.isNotEmpty(fields)
? ImmutableSortedSet.copyOf(fields)
: ImmutableSortedSet.of(UserProfile.Fields.ISSUER,
UserProfile.Fields.TITLE,
UserProfile.Fields.DESCRIPTION,
UserProfile.Fields.ADDRESS,
UserProfile.Fields.CITY);
List<Feature> rows = Beans.getStream(profiles)
.map(profile -> {
Feature feature = new Feature();
// Set properties
feature.setProperties(fieldList.stream()
.map(property -> {
Object value = Beans.getProperty(profile, property);
if (value == null) return null; // Will be filtered
return new AbstractMap.SimpleEntry<>(property, value);
})
.filter(Objects::nonNull) // Avoid null value
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))
);
// Set geometry
GeoJsonObject geometry = Geometries.toGeometry(profile.getGeoPoint());
feature.setGeometry(geometry);
return feature;
})
.filter(feature -> feature.getGeometry() != null)
.collect(Collectors.toList());
features.setFeatures(rows);
return features;
}
protected void fillDefaults(Wallet wallet, UserProfile userProfile) {
super.fillDefaults(wallet, userProfile);
// Workaround, to clear avatar - Is it a bug in the ES attachment-mapper ?
if (userProfile.getAvatar() == null) {
userProfile.setAvatar(Attachment.builder()
.content("")
.contentType("")
.build());
}
}
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import lombok.NonNull;
import org.duniter.core.beans.Service;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.client.model.filter.UserSettingsFilter;
import org.duniter.elasticsearch.model.user.UserSettings;
import javax.annotation.Nullable;
import java.util.Optional;
import java.util.stream.Stream;
public interface UserSettingsService extends Service {
Stream<UserSettings> findAllByFilter(@NonNull Peer peer,
@NonNull UserSettingsFilter filter,
@Nullable Page page);
UserSettings save(Peer peer, Wallet wallet, UserSettings settings);
Optional<UserSettings> findByPubkey(Peer peer, String pubkey);
boolean deleteByPubkey(Peer peer, Wallet wallet, String pubkey);
boolean delete(Peer peer, Wallet wallet, UserSettings settings);
}
package org.duniter.elasticsearch.client.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpEntity;
import org.apache.http.client.entity.EntityBuilder;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.core.client.service.exception.HttpNotFoundException;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ArrayUtils;
import org.duniter.core.util.Preconditions;
import org.duniter.core.util.StringUtils;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.client.model.filter.UserSettingsFilter;
import org.duniter.elasticsearch.client.model.query.*;
import org.duniter.elasticsearch.dao.user.IUserSettingsRepository;
import org.duniter.elasticsearch.model.DeleteRecord;
import org.duniter.elasticsearch.model.query.SearchResponse;
import org.duniter.elasticsearch.model.user.UserSettings;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Stream;
@Slf4j
public class UserSettingsServiceImpl extends AbstractServiceImpl implements UserSettingsService {
protected Configuration config;
public interface Uris {
String SEARCH = "/user/settings/_search";
String GET_SOURCE_BY_PUBKEY = "/user/settings/%s/_source";
String POST_INSERT = "/user/settings";
String POST_DELETE = "/history/delete";
}
@Override
public void afterPropertiesSet() {
super.afterPropertiesSet();
config = Configuration.instance();
}
@Override
public void close() throws IOException {
super.close();
}
@Override
public Stream<UserSettings> findAllByFilter(@NonNull Peer peer,
@NonNull UserSettingsFilter filter,
@Nullable Page page) {
page = Page.nullToDefault(page);
ObjectMapper objectMapper = getObjectMapper();
try {
List<SearchQuery> filters = Lists.newArrayList();
List<SearchQuery> matches = Lists.newArrayList();
if (ArrayUtils.isNotEmpty(filter.getIssuers())) {
filters.add(SearchQueries
.terms(UserSettings.Fields.ISSUER, filter.getIssuers())
);
}
// Query String
if (StringUtils.isNotBlank(filter.getQueryString())) {
matches.add(SearchQueries.queryString(filter.getQueryString()));
}
SearchQuery query = SearchQuery.builder()
.bool(BoolQuery.builder()
.must(matches.toArray(new SearchQuery[0]))
.filter(filters.toArray(new SearchQuery[0]))
.build())
.build();
SearchRequest request = SearchRequest.builder()
.query(query)
.from(page.getFrom())
.size(page.getSize())
.source(filter.getFields())
.sortBy(page.getSortBy(), page.getSortDirection())
.build();
long now = System.currentTimeMillis();
if (log.isDebugEnabled()) {
log.debug("Searching on user profiles using: {}", objectMapper.writeValueAsString(request));
}
HttpPost httpPost = new HttpPost(httpService.getPath(peer, Uris.SEARCH));
HttpEntity entity = EntityBuilder.create()
.setContentType(ContentType.APPLICATION_JSON)
.setBinary(objectMapper.writeValueAsBytes(request))
.build();
httpPost.setEntity(entity);
SearchResponse response = httpService.executeRequest(httpPost, SearchResponse.class);
if (log.isDebugEnabled()) {
log.debug("Searching on user settings [OK] - {} result(s) found in {}ms",
ArrayUtils.size(response.getHits().getHits()),
System.currentTimeMillis() - now);
}
return Arrays.stream(response.getHits().getHits())
.map(hit -> {
try {
return objectMapper.treeToValue(hit.getSource(), UserSettings.class);
} catch (JsonProcessingException e) {
log.warn("Error during user settings deserialization");
return null;
}
})
.filter(Objects::nonNull);
} catch (JsonProcessingException e) {
throw new TechnicalException(e);
}
}
@Override
public UserSettings save(Peer peer, Wallet wallet, UserSettings userSettings) {
fillDefaults(wallet, userSettings);
return sendRecord(peer, Uris.POST_INSERT, wallet, userSettings);
}
@Override
public Optional<UserSettings> findByPubkey(Peer peer, String pubkey) {
try {
UserSettings settings = httpService.executeRequest(peer,
String.format(Uris.GET_SOURCE_BY_PUBKEY, pubkey),
UserSettings.class);
return Optional.of(settings);
}
catch (HttpNotFoundException e) {
return Optional.empty(); // Not found
}
}
@Override
public boolean delete(@NonNull Peer peer, @NonNull Wallet wallet, @NonNull UserSettings userSettings) {
Preconditions.checkNotNull(userSettings.getIssuer());
return deleteByPubkey(peer, wallet, userSettings.getIssuer());
}
@Override
public boolean deleteByPubkey(Peer peer, Wallet wallet, String pubkey) {
DeleteRecord record = new DeleteRecord();
record.setIndex(IUserSettingsRepository.INDEX);
record.setType(IUserSettingsRepository.TYPE);
record.setId(pubkey);
DeleteRecord savedRecord = sendRecord(peer, Uris.POST_DELETE, wallet, record);
return StringUtils.isNotBlank(savedRecord.getId());
}
}
org.duniter.core.client.service.bma.BlockchainRemoteServiceImpl
org.duniter.core.client.service.bma.NetworkRemoteServiceImpl
org.duniter.core.client.service.bma.WotRemoteServiceImpl
org.duniter.core.client.service.bma.TransactionRemoteServiceImpl
org.duniter.core.client.service.elasticsearch.CurrencyPodRemoteServiceImpl
org.duniter.core.service.Ed25519CryptoServiceImpl
org.duniter.core.client.service.HttpServiceImpl
org.duniter.core.client.service.DataContext
org.duniter.core.client.service.local.PeerServiceImpl
org.duniter.core.client.service.local.CurrencyServiceImpl
org.duniter.core.client.service.local.NetworkServiceImpl
org.duniter.core.client.repositories.mem.MemoryCurrencyRepositoryImpl
org.duniter.core.client.repositories.mem.MemoryPeerRepositoryImpl
org.duniter.elasticsearch.client.service.UserProfileServiceImpl
org.duniter.elasticsearch.client.service.UserSettingsServiceImpl
\ No newline at end of file
### ###
# Global logging configuration # Global logging configuration
log4j.rootLogger=ERROR, stdout, file log4j.rootLogger=INFO, stdout, file
# Console output # Console appender
log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %5p (%c:%L) - %m%n log4j.appender.stdout.layout.ConversionPattern=%p - %m%n
# ucoin levels
log4j.logger.org.duniter=DEBUG
log4j.logger.org.duniter.core.client.service.bma.AbstractNetworkService=WARN
# File appender
log4j.appender.file=org.apache.log4j.RollingFileAppender log4j.appender.file=org.apache.log4j.RollingFileAppender
log4j.appender.file.file=ucoin-client.log log4j.appender.file.file=target/cesium-plus-pod-client.log
log4j.appender.file.MaxFileSize=10MB log4j.appender.file.MaxFileSize=10MB
log4j.appender.file.MaxBackupIndex=4 log4j.appender.file.MaxBackupIndex=4
log4j.appender.file.layout=org.apache.log4j.PatternLayout log4j.appender.file.layout=org.apache.log4j.PatternLayout
log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p %c - %m%n log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p %c - %m%n
# Duniter4j levels
log4j.logger.org.duniter=WARN
# Cesium+ Pod levels
log4j.logger.org.duniter.elasticsearch.client=DEBUG
log4j.logger.org.duniter.cesium.pod=DEBUG
# Other frameworks levels
log4j.logger.org.apache.http=ERROR
log4j.logger.org.nuiton.util=WARN
log4j.logger.org.nuiton.config=WARN
log4j.logger.org.nuiton.converter=WARN
log4j.logger.org.nuiton.i18n=ERROR
# Http client connection debug
#log4j.logger.org.apache.http.impl.conn=DEBUG
\ No newline at end of file
package org.duniter.core.client; package org.duniter.cesium.pod;
/* /*-
* #%L * #%L
* UCoin Java :: Core Client API * Cesium+ pod :: Client API
* %% * %%
* Copyright (C) 2014 - 2016 EIS * Copyright (C) 2014 - 2023 Duniter Team
* %% * %%
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as * it under the terms of the GNU Affero General Public License as published by
* published by the Free Software Foundation, either version 3 of the * the Free Software Foundation, either version 3 of the License, or
* License, or (at your option) any later version. * (at your option) any later version.
* *
* This program is distributed in the hope that it will be useful, * This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details. * GNU General Public License for more details.
* *
* You should have received a copy of the GNU General Public * You should have received a copy of the GNU Affero General Public License
* License along with this program. If not, see * along with this program. If not, see <http://www.gnu.org/licenses/>.
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L% * #L%
*/ */
public class TestFixtures extends org.duniter.core.test.TestFixtures { public class TestFixtures extends org.duniter.core.test.TestFixtures {
public String getDefaultCurrency() {
public long getDefaultCurrencyId() { return "g1";
return -1;
} }
} }
package org.duniter.core.client; package org.duniter.cesium.pod;
/* /*-
* #%L * #%L
* UCoin Java Client :: Core API * Cesium+ pod :: Client API
* %% * %%
* Copyright (C) 2014 - 2015 EIS * Copyright (C) 2014 - 2023 Duniter Team
* %% * %%
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as * it under the terms of the GNU Affero General Public License as published by
* published by the Free Software Foundation, either version 3 of the * the Free Software Foundation, either version 3 of the License, or
* License, or (at your option) any later version. * (at your option) any later version.
* *
* This program is distributed in the hope that it will be useful, * This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of * but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details. * GNU General Public License for more details.
* *
* You should have received a copy of the GNU General Public * You should have received a copy of the GNU Affero General Public License
* License along with this program. If not, see * along with this program. If not, see <http://www.gnu.org/licenses/>.
* <http://www.gnu.org/licenses/gpl-3.0.html>.
* #L% * #L%
*/ */
import com.google.common.collect.Lists; import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.duniter.core.client.config.Configuration; import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.config.ConfigurationOption; import org.duniter.core.client.config.ConfigurationOption;
import org.duniter.core.client.model.local.Peer; import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.service.ServiceLocator; import org.duniter.core.client.service.ServiceLocator;
import org.apache.commons.io.FileUtils;
import org.junit.runner.Description; import org.junit.runner.Description;
import org.nuiton.i18n.I18n; import org.nuiton.i18n.I18n;
import org.nuiton.i18n.init.DefaultI18nInitializer; import org.nuiton.i18n.init.DefaultI18nInitializer;
import org.nuiton.i18n.init.UserI18nInitializer; import org.nuiton.i18n.init.UserI18nInitializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File; import java.io.File;
import java.io.IOException; import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.Locale; import java.util.Locale;
@Slf4j
public class TestResource extends org.duniter.core.test.TestResource { public class TestResource extends org.duniter.core.test.TestResource {
private static final Logger log = LoggerFactory.getLogger(TestResource.class);
public static TestResource create() { public static TestResource create() {
return new TestResource(null); return new TestResource(null);
} }
...@@ -63,6 +59,11 @@ public class TestResource extends org.duniter.core.test.TestResource { ...@@ -63,6 +59,11 @@ public class TestResource extends org.duniter.core.test.TestResource {
return fixtures; return fixtures;
} }
@Override
public File getResourceDirectory() {
return super.getResourceDirectory();
}
protected void before(Description description) throws Throwable { protected void before(Description description) throws Throwable {
super.before(description); super.before(description);
...@@ -92,20 +93,19 @@ public class TestResource extends org.duniter.core.test.TestResource { ...@@ -92,20 +93,19 @@ public class TestResource extends org.duniter.core.test.TestResource {
* @return the prefix to use to retrieve configuration files * @return the prefix to use to retrieve configuration files
*/ */
protected String getConfigFilesPrefix() { protected String getConfigFilesPrefix() {
return "duniter4j-core-client-test"; return "cesium-plus-pod-client-test";
} }
protected String getI18nBundleName() { protected String getI18nBundleName() {
return "duniter4j-core-client-i18n"; return "cesium-plus-pod-client-i18n";
} }
/* -- -- */ /* -- -- */
/** /**
* Convenience methods that could be override to initialize other configuration * Convenience methods that could be overridden to initialize other configuration
* *
* @param configFilename * @param configFilename
* @param configArgs
*/ */
protected void initConfiguration(String configFilename) { protected void initConfiguration(String configFilename) {
String[] configArgs = getConfigArgs(); String[] configArgs = getConfigArgs();
...@@ -133,30 +133,30 @@ public class TestResource extends org.duniter.core.test.TestResource { ...@@ -133,30 +133,30 @@ public class TestResource extends org.duniter.core.test.TestResource {
Locale i18nLocale = config.getI18nLocale(); Locale i18nLocale = config.getI18nLocale();
if (log.isInfoEnabled()) { if (log.isDebugEnabled()) {
log.info(String.format("Starts i18n with locale [%s] at [%s]", log.debug(String.format("Starts i18n with locale [%s] at [%s]",
i18nLocale, i18nDirectory)); i18nLocale, i18nDirectory));
} }
I18n.init(new UserI18nInitializer( I18n.init(new UserI18nInitializer(
i18nDirectory, new DefaultI18nInitializer(getI18nBundleName())), i18nDirectory, new DefaultI18nInitializer(getI18nBundleName())),
i18nLocale); i18nLocale);
} }
protected String[] getConfigArgs() { protected String[] getConfigArgs() {
List<String> configArgs = Lists.newArrayList(); List<String> configArgs = Lists.newArrayList();
configArgs.addAll(Lists.newArrayList( configArgs.addAll(Lists.newArrayList(
"--option", ConfigurationOption.BASEDIR.getKey(), getResourceDirectory().getAbsolutePath())); "--option", ConfigurationOption.BASEDIR.getKey(), getResourceDirectory().getAbsolutePath()));
return configArgs.toArray(new String[configArgs.size()]); return configArgs.toArray(new String[configArgs.size()]);
} }
protected void initMockData() { protected void initMockData() {
Configuration config = Configuration.instance(); Configuration config = Configuration.instance();
// Set a default account id, then load cache Peer peer = Peer.builder()
ServiceLocator.instance().getDataContext().setAccountId(0); .host(config.getNodeHost())
.port(config.getNodePort())
Peer peer = new Peer(config.getNodeHost(), config.getNodePort()); .build();
peer.setCurrencyId(fixtures.getDefaultCurrencyId()); peer.setCurrency(fixtures.getDefaultCurrency());
ServiceLocator.instance().getPeerService().save(peer); ServiceLocator.instance().getPeerService().save(peer);
......
package org.duniter.cesium.pod.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import org.duniter.cesium.pod.TestResource;
import org.duniter.core.beans.Service;
import org.duniter.core.client.config.Configuration;
import org.duniter.core.client.model.local.Peer;
import org.duniter.core.client.model.local.Wallet;
import org.duniter.core.client.service.HttpService;
import org.duniter.elasticsearch.client.service.ServiceLocator;
import org.duniter.elasticsearch.client.service.UserProfileService;
import org.junit.Assume;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Test;
@Slf4j
public abstract class AbstractServiceTest<S extends Service> {
private final Class<S> serviceClass;
protected S service;
protected Peer peer;
protected Wallet wallet;
protected Configuration configuration;
protected AbstractServiceTest(Class<S> serviceClass) {
this.serviceClass = serviceClass;
}
@Before
public void setUp() {
service = ServiceLocator.instance().getBean(serviceClass);
Assume.assumeNotNull(service);
configuration = Configuration.instance();
Assume.assumeNotNull(configuration);
}
protected Peer getPeer(String currency) {
return ServiceLocator.instance().getPeerService().getActivePeerByCurrency(currency);
}
protected Wallet getWallet(String currency, String uid, String pubkey, String seckey) {
return new Wallet(currency, uid, pubkey, seckey);
}
}
package org.duniter.cesium.pod.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.duniter.cesium.pod.TestResource;
import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.client.service.exception.HttpNotFoundException;
import org.duniter.core.client.service.exception.HttpTimeoutException;
import org.duniter.core.client.service.exception.HttpUnauthorizeException;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ArrayUtils;
import org.duniter.core.util.Beans;
import org.duniter.core.util.CollectionUtils;
import org.duniter.core.util.StringUtils;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.model.SortDirection;
import org.duniter.elasticsearch.client.model.filter.MovementFilter;
import org.duniter.elasticsearch.client.model.filter.UserProfileFilter;
import org.duniter.elasticsearch.client.model.filter.UserSettingsFilter;
import org.duniter.elasticsearch.client.model.geom.Envelope;
import org.duniter.elasticsearch.client.service.ServiceLocator;
import org.duniter.elasticsearch.client.service.UserProfileService;
import org.duniter.elasticsearch.client.service.UserSettingsService;
import org.duniter.elasticsearch.model.blockchain.Movement;
import org.duniter.elasticsearch.model.type.Attachment;
import org.duniter.elasticsearch.model.type.GeoPoint;
import org.duniter.elasticsearch.model.user.UserProfile;
import org.duniter.elasticsearch.model.user.UserSettings;
import org.duniter.elasticsearch.utils.Dates;
import org.duniter.elasticsearch.utils.Geometries;
import org.junit.*;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.text.ParseException;
import java.util.*;
import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Slf4j
public class FakeUserProfilesTest extends AbstractServiceTest<UserProfileService>{
@ClassRule
public static final TestResource resource = TestResource.create();
private UserSettingsService settingsService;
public FakeUserProfilesTest(){
super(UserProfileService.class);
}
@Before
public void setUp() {
super.setUp();
settingsService = ServiceLocator.instance().getUserSettingsService();
peer = getPeer(resource.getFixtures().getDefaultCurrency());
wallet = getWallet(resource.getFixtures().getDefaultCurrency(),
null,
resource.getFixtures().getUserPublicKey(),
resource.getFixtures().getUserSecretKey()
);
}
@Test
public void deleteDicoProfile() throws IOException, ParseException {
// Load dictionary, from fake profiles
Set<String> dictionary = createDictionary(ImmutableList.<UserProfile>builder()
.addAll(loadProfilesFile(new File("src/test/resources/fake-profiles-543.json")))
.addAll(loadProfilesFile(new File("src/test/resources/fake-profiles-4600.json")))
.addAll(loadProfilesFile(new File("src/test/resources/fake-profiles-G.json")))
.build());
List<UserProfile> profiles;
List<UserProfile> rejectedProfiles = Lists.newArrayList();
String filePathPattern = "target/similar-profiles-%s.json";
File inputProfilesFile = new File(String.format(filePathPattern, "input"));
if (inputProfilesFile.exists()) {
profiles = loadProfilesFile(inputProfilesFile);
}
else {
profiles = Lists.newArrayList();
Date startDate = Dates.parseDate("2021-06-30", "yyy-MM-dd");
long now = System.currentTimeMillis();
long sliceDuration = 2 * 30 * 24 * 60 * 60 * 1000; // 2 months, in millis
for (long startTime = startDate.getTime(); startTime < now; startTime += sliceDuration) {
long endTime = Math.min(now, startTime + sliceDuration);
profiles.addAll(loadProfiles(UserProfileFilter.builder()
.startDate(new Date(startTime))
.endDate(new Date(endTime))
.build()));
}
log.info("Found {} profiles", profiles.size());
// Filter no settings
profiles = filterNoSettings(profiles, rejectedProfiles);
// Filter no Tx
profiles = filterNoTx(profiles, rejectedProfiles);
dumpToFile(profiles, inputProfilesFile, true);
dumpToGeoJsonFile(profiles, new File(inputProfilesFile.getPath() + ".geojson"), true);
}
// Force dictionary profile
//profiles = filterDictionary(profiles, dictionary);
// Filter no description
//profiles = profiles.stream().filter(p -> StringUtils.isBlank(p.getDescription())).collect(Collectors.toList());
// Filter no address
//profiles = profiles.stream().filter(p -> StringUtils.isBlank(p.getAddress())).collect(Collectors.toList());
// Filter no postal code
//profiles = profiles.stream().filter(p -> StringUtils.isNotBlank(p.getCity())
// && !p.getCity().contains("0") && !p.getCity().contains("1"))
// .collect(Collectors.toList());
// Filter no geo point
//profiles = profiles.stream().filter(p -> p.getGeoPoint() == null)
// .collect(Collectors.toList());
//dumpToGeoJsonFile(profiles, new File(inputProfilesFile.getPath() + ".geojson"), true);
//dumpToFile(profiles, inputProfilesFile, true);
double startDetectionThreshold = 70;
double thresholdStep = 10;
double minGroupSize = 20;
Map<Integer, Integer> profileCountByHash = Maps.newLinkedHashMap();
Map<Integer, File> fileByHash = Maps.newLinkedHashMap();
List<List<UserProfile>> inputs = Lists.newArrayList();
inputs.add(profiles);
int groupIndex = 0;
while (!inputs.isEmpty()) {
List<UserProfile> loopInput = inputs.remove(0);
double detectionThreshold = startDetectionThreshold;
while (detectionThreshold < 100) {
List<UserProfile> loopRejectedProfiles = Lists.newArrayList();
File groupFile = new File(String.format(filePathPattern, groupIndex));
File groupGeoFile = new File(String.format(filePathPattern, groupIndex) + ".geojson");
// Find similar profiles
profiles = filterSimilarProfiles(loopInput, loopRejectedProfiles, dictionary, detectionThreshold, thresholdStep);
// Compute a unique hash, for the group
int hash = computeProfileHashCode(profiles);
if (!profileCountByHash.containsKey(hash) && profiles.size() > 0) {
if (profiles.size() > minGroupSize) {
// Dump to files
dumpToGeoJsonFile(profiles, groupGeoFile, true);
dumpToFile(profiles, groupFile, true);
// Add to hash map
profileCountByHash.put(hash, profiles.size());
fileByHash.put(hash, groupFile);
groupIndex++;
}
// Will loop on rejected profiles
int rejectedHash = computeProfileHashCode(loopRejectedProfiles);
if (!profileCountByHash.containsKey(rejectedHash) && loopRejectedProfiles.size() > minGroupSize) {
inputs.add(loopRejectedProfiles);
}
}
detectionThreshold += thresholdStep;
}
}
if (!fileByHash.isEmpty()) {
log.warn("Found {} groups of similar profiles:", fileByHash.size());
profileCountByHash.entrySet().forEach(e -> {
log.warn(" - {} profiles (hash: {}) at {}", e.getValue(), e.getKey(), fileByHash.get(e.getKey()));
});
}
}
public List<UserProfile> filterSimilarProfiles(List<UserProfile> profiles, List<UserProfile> rejectedProfiles,
Set<String> dictionary,
double startDetectionThreshold,
double thresholdStep) {
log.info("--- Analysing {} profiles...", profiles.size());
// Analyse fields
double detectionThreshold = startDetectionThreshold;
int counter = 0;
while (detectionThreshold <= 100) {
int startSize = profiles.size();
log.info("-- Pass n°{}: {} profiles - detection threshold: {}%", ++counter, profiles.size(), detectionThreshold);
// Analyze fields
profiles = analyseAndFilterOnFields(profiles, rejectedProfiles, dictionary, detectionThreshold, "yyyy-MM", false);
int delta = startSize - profiles.size();
if (delta == 0) {
if (detectionThreshold == startDetectionThreshold) break; // Not need to try again
// Return to the start threshold
detectionThreshold = startDetectionThreshold;
}
else {
detectionThreshold += thresholdStep;
}
}
log.info("--- Find {} similar profiles", profiles.size());
return profiles;
}
@Test
public void findAndDeleteFakeProfiles_543() {
List<UserProfile> profiles = loadProfiles(UserProfileFilter.builder()
// Insertion period
.startDate(Dates.safeParseDate("2021-06-29 23:00", "yyyy-MM-dd HH:mm"))
.endDate(Dates.safeParseDate("2021-06-30 03:00", "yyyy-MM-dd HH:mm"))
.build(), UserProfile.Fields.TIME, SortDirection.ASC);
Assume.assumeNotNull(profiles);
Assume.assumeTrue(profiles.size() > 0);
Assume.assumeTrue(profiles.size() >= 540);
List<UserProfile> rejectedProfiles = Lists.newArrayList(); // Rejected = valid profiles (not fake)
// Analyse settings presence
profiles = filterNoSettings(profiles, rejectedProfiles);
// Analyse movement presence
profiles = filterNoTx(profiles, rejectedProfiles);
// Analyse fields
profiles = analyseAndFilterOnFields(profiles, rejectedProfiles, 90 /*90%*/, false);
log.info("Find {} fake profiles", profiles.size());
Assume.assumeTrue("Nothing to delete", profiles.size() > 0);
// Apply deletion
deleteProfiles(profiles);
}
@Test
public void findAndDeleteFakeProfiles_4600() {
List<UserProfile> profiles = loadProfiles(UserProfileFilter.builder()
// Insertion period
.startDate(Dates.safeParseDate("2021-06-30 22:00", "yyyy-MM-dd HH:mm"))
.endDate(Dates.safeParseDate("2021-07-02", "yyyy-MM-dd"))
.build());
Assume.assumeNotNull(profiles);
Assume.assumeTrue(profiles.size() > 0);
Assume.assumeTrue(profiles.size() >= 4600);
List<UserProfile> rejectedProfiles = Lists.newArrayList(); // Rejected = valid profiles (not fake)
// Analyse settings presence
profiles = filterNoSettings(profiles, rejectedProfiles);
// Analyse movement presence
profiles = filterNoTx(profiles, rejectedProfiles);
// Analyse fields
profiles = analyseAndFilterOnFields(profiles, rejectedProfiles, 90 /*90%*/, true);
log.info("Find {} fake profiles", profiles.size());
Assume.assumeTrue("Nothing to delete", profiles.size() > 0);
// Apply deletion
deleteProfiles(profiles);
}
@Test
@Ignore
public void findAndDeleteFakeProfiles_G() {
List<UserProfile> profiles = loadProfiles(UserProfileFilter.builder()
// Should match some profiles (bbox France)
.boundingBox(Envelope.builder()
.minX(0).maxX(3)
.minY(44).maxY(47.9)
.build())
// Insertion period
.startDate(Dates.safeParseDate("2021-07-01", "yyyy-MM-dd"))
.endDate(Dates.safeParseDate("2021-07-02", "yyyy-MM-dd"))
.build());
Assume.assumeNotNull(profiles);
Assume.assumeTrue(profiles.size() > 0);
List<UserProfile> rejectedProfiles = Lists.newArrayList();
List<UserProfile> doubtfulProfiles = Lists.newArrayList();
// Analyse fields
profiles = analyseAndFilterOnFields(profiles, rejectedProfiles, 90 /*90%*/, false);
// Analyse distance between point
profiles = analyseAndFilterOnDistance(profiles, rejectedProfiles, doubtfulProfiles,true);
log.info("Find {} fake profiles", profiles.size());
Assume.assumeTrue("Nothing to delete", profiles.size() > 0);
// Apply deletion
//deleteProfiles(profiles);
}
@Test
@Ignore
public void deleteProfilesFromFile() throws IOException {
List<UserProfile> profiles = loadProfilesFile(new File("src/test/resources/fake-profiles-G.json"));
Assert.assertTrue(profiles.size() == 5314);
List<UserProfile> rejectedProfiles = Lists.newArrayList();
// Make sure profiles exists
profiles = filterExists(profiles, rejectedProfiles);
log.info("Find {} existing profiles", profiles.size());
Assume.assumeTrue("Nothing to delete", profiles.size() > 0);
// Apply deletion
//deleteProfiles(profiles);
}
protected List<UserProfile> analyseAndFilterOnFields(List<UserProfile> profiles,
List<UserProfile> rejectedProfiles,
double filterThresholdPct,
boolean outputToFile) {
return analyseAndFilterOnFields(profiles, rejectedProfiles, null, filterThresholdPct,
"yyyy-MM-dd", outputToFile);
}
protected List<UserProfile> analyseAndFilterOnFields(List<UserProfile> profiles,
List<UserProfile> rejectedProfiles,
Set<String> dictionary,
double filterThresholdPct,
String timePattern,
boolean outputToFile) {
// Analyse avatar
{
profiles = analyseAndFilterOnField(profiles, rejectedProfiles, UserProfile.Fields.AVATAR,
(Attachment avatar) -> avatar != null && StringUtils.isNotBlank(avatar.getContentType())
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Analyse description
{
profiles = analyseAndFilterOnField(profiles, rejectedProfiles, UserProfile.Fields.DESCRIPTION,
(String description) -> StringUtils.isNotBlank(description)
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Analyse address
{
// Exists or not
profiles = analyseAndFilterOnField(profiles, rejectedProfiles, UserProfile.Fields.ADDRESS,
(String value) -> StringUtils.isNotBlank(value)
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Analyse city
{
// Exists or not
profiles = analyseAndFilterOnField(profiles, rejectedProfiles, UserProfile.Fields.CITY,
(String city) -> StringUtils.isNotBlank(city)
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
// With postal code
profiles = analyseAndFilterOnField(profiles, rejectedProfiles,
"withPostalCode",
UserProfile::getCity,
(String city) -> this.hasPostalCode(city)
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Analyse date
{
profiles = analyseAndFilterOnField(profiles, rejectedProfiles,
UserProfile.Fields.TIME,
(Long time) -> time == null || time == 0d ? "null" : Dates.formatDate(new Date(time * 1000), timePattern),
filterThresholdPct
);
}
// Create the dictionary
if (dictionary == null){
int wordNbFoundThreshold = 3;
Map<String, Integer> counterMap = Maps.newHashMap();
profiles.stream()
.map(UserProfile::getTitle)
.flatMap(this::streamWords)
.forEach(value -> {
Integer counter = counterMap.get(value);
counterMap.put(value, (counter != null ? counter : 0) + 1);
});
dictionary = counterMap.keySet().stream()
.filter(value -> counterMap.get(value) >= wordNbFoundThreshold)
.filter(value -> value.length() > 2) // Too short words
.filter(value -> !value.matches("[A-Z][A-Z]+")) // Capitalize word (family name)
.collect(Collectors.toSet());
log.info(" - Word dictionary (appearing at least in {} profiles): {}", wordNbFoundThreshold, dictionary);
}
// Analyse title words
if (dictionary.size() > 0) {
Set<String> finalDictionary = dictionary;
profiles = analyseAndFilterOnField(profiles, rejectedProfiles,
"dictionaryWords",
UserProfile::getTitle,
(String title) -> streamWords(title).anyMatch(finalDictionary::contains) ? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Analyse geoPoint
{
profiles = analyseAndFilterOnField(profiles, rejectedProfiles, UserProfile.Fields.GEO_POINT,
(GeoPoint point) -> point != null && point.getLon() != null && point.getLat() != null
? Boolean.TRUE : Boolean.FALSE,
filterThresholdPct
);
}
// Dump to files
if (outputToFile) {
log.info("Creating output files...");
try {
// Write fake profiles into GeoJson file
dumpToFile(profiles, new File("target", "fake-profiles-G.json"), true);
dumpToFile(rejectedProfiles, new File("target", "rejected-profiles.json"), true);
}
catch (IOException e) {
Assume.assumeNoException(e);
}
}
return profiles;
}
protected List<UserProfile> analyseAndFilterOnDistance(List<UserProfile> profiles,
List<UserProfile> rejectedProfiles,
List<UserProfile> doubtfulProfiles,
boolean outputToFile) {
// Parameter for distance rule detection
double minDistanceMeters = 1750;
double maxDistanceMeters = 1865;
final List<UserProfile> profilesCopy = ImmutableList.copyOf(profiles);
log.error("Analyzing distance between geo points...");
Map<Long, Integer> counterMap = Maps.newHashMap();
profiles = profiles.stream().filter((profile) -> {
double lon = round(profile.getGeoPoint().getLon(), 5);
double lat = round(profile.getGeoPoint().getLat(), 5);
long minNeighborDistance = -1;
int neighborsCounter = 0;
for (UserProfile other : profilesCopy) {
if (!Objects.equals(other.getIssuer(), profile.getIssuer())) {
double otherLon = round(other.getGeoPoint().getLon(), 5);
double otherLat = round(other.getGeoPoint().getLat(), 5);
long distance = Geometries.getDistanceInMeters(lon, lat, otherLon, otherLat);
boolean isNeighbor = distance >= minDistanceMeters && distance <= maxDistanceMeters;
if (isNeighbor) {
neighborsCounter++;
if (minNeighborDistance == -1 || distance < minNeighborDistance) {
minNeighborDistance = distance;
}
}
}
}
if (minNeighborDistance != -1) {
// Round at 100 meters
minNeighborDistance = Math.round((1d * minNeighborDistance / 100) + 0.5) * 100;
Integer counter = counterMap.get(minNeighborDistance);
counterMap.put(minNeighborDistance, (counter != null ? counter : 0) + 1);
}
// No neighbor
if (neighborsCounter == 0) {
// Check if settings exists
UserSettings settings = settingsService.findByPubkey(peer, profile.getIssuer()).orElse(null);
if (settings != null) {
rejectedProfiles.add(profile);
return false; // Keep it, if settings exists
}
// Check if movements
boolean hasMovements = service.findMovements(peer, MovementFilter.builder()
.pubkey(profile.getIssuer()).build(),
Page.builder().size(1).build())
.anyMatch(Objects::nonNull);
if (hasMovements) {
rejectedProfiles.add(profile);
return false; // Keep it, if movements exists
}
// Log to understand why there is no neighbor
/*try {
log.debug("Detected doubtful profile: no neighbor BUT no settings and NO movements: {} - {}", objectMapper.writeValueAsString(profile));
} catch (Exception e) {} // Silent
doubtfulProfiles.add(profile);
return false;*/
}
return true;
})
.collect(Collectors.toList());
// DEBUG - Log all neightbor distance found
counterMap.keySet().stream().sorted()
.forEach(distance -> {
int counter = counterMap.get(distance);
//if (counter < 10)
log.info("distance={} counter={}", distance, counter);
});
// Find figure center
{
double minLat = profiles.stream()
.map(profile -> profile.getGeoPoint().getLat())
.reduce(9999d, Math::min);
double maxLat = profiles.stream()
.map(profile -> profile.getGeoPoint().getLat())
.reduce(-9999d, Math::max);
double minLon = profiles.stream()
.map(profile -> profile.getGeoPoint().getLon())
.reduce(9999d, Math::min);
double maxLon = profiles.stream()
.map(profile -> profile.getGeoPoint().getLon())
.reduce(-9999d, Math::max);
double centerLat = minLat + (maxLat - minLat) / 2;
double centerLon = minLon + (maxLon - minLon) / 2;
// https://www.google.com/maps/@45.93681064446829,1.8011600876707416,16z
log.info("Figure center: https://www.google.com/maps/@{},{},16z", centerLat, centerLon);
}
// Dump to files
if (outputToFile) {
log.info("Creating output files...");
try {
// Write fake profiles into GeoJson file
dumpToGeoJsonFile(profiles, new File("target", "fake-profiles-G.geojson"), true);
dumpToFile(profiles, new File("target", "fake-profiles-G.json"), true);
dumpToGeoJsonFile(doubtfulProfiles, new File("target", "doubtful-profiles.geojson"), true);
dumpToGeoJsonFile(rejectedProfiles, new File("target", "rejected-profiles.geojson"), true);
}
catch (IOException e) {
Assume.assumeNoException(e);
}
}
return profiles;
}
protected List<UserProfile> filterExists(@NonNull List<UserProfile> profiles, @NonNull List<UserProfile> rejectedProfiles) {
// Make sure profiles exists
Map<String, UserProfile> profileByPubkeys = Beans.splitByProperty(profiles, UserProfile.Fields.ISSUER);
String[] pubkeys = profileByPubkeys.keySet().toArray(new String[0]);
int size = 1000;
List<UserProfile> existingProfiles = Lists.newArrayList();
for (int start = 0; start< pubkeys.length; start+= size) {
int end = Math.min(start+size, pubkeys.length);
service.findAllByFilter(peer, UserProfileFilter.builder()
.issuers(Arrays.copyOfRange(pubkeys, start, end))
.fields(new String[]{UserProfile.Fields.ISSUER})
.build(), null)
.map(UserProfile::getIssuer)
.map(profileByPubkeys::remove)
.filter(Objects::nonNull)
.forEach(existingProfiles::add);
}
if (!profileByPubkeys.isEmpty()) {
rejectedProfiles.addAll(profileByPubkeys.values());
}
return existingProfiles;
}
protected List<UserProfile> filterNoSettings(@NonNull List<UserProfile> profiles, @NonNull List<UserProfile> rejectedProfiles) {
log.info("Filtering on 'without settings'...");
Map<String, UserProfile> profilesByPubkeys = Beans.splitByProperty(profiles, UserProfile.Fields.ISSUER);
String[] pubkeys = profilesByPubkeys.keySet().toArray(new String[0]);
int fetchSize = 100;
for (int i = 0; i < pubkeys.length; i += fetchSize) {
//log.debug("Check user profile's settings' {}/{}...", i, pubkeys.length);
int end = Math.min(pubkeys.length, i+fetchSize);
UserSettingsFilter filter = UserSettingsFilter.builder()
.issuers(Arrays.copyOfRange(pubkeys, i, end))
.fields(new String[]{UserSettings.Fields.ISSUER, UserSettings.Fields.TIME})
.build();
settingsService.findAllByFilter(peer, filter, null)
.map(UserSettings::getIssuer)
// Remove from the result map
.map(profilesByPubkeys::remove)
.filter(Objects::nonNull)
// Add removed profile to the reject list
.forEach(rejectedProfiles::add);
}
List<UserProfile> profilesNoSettings = Lists.newArrayList(profilesByPubkeys.values());
int delta = profiles.size() - profilesNoSettings.size();
log.info("Found {} profiles without settings ({} has settings}", profilesNoSettings.size(), delta);
return profilesNoSettings;
}
protected List<UserProfile> filterNoTx(@NonNull List<UserProfile> profiles, @NonNull List<UserProfile> rejectedProfiles) {
log.info("Filtering on 'without transaction'...");
Map<String, UserProfile> profilesByPubkeys = Beans.splitByProperty(profiles, UserProfile.Fields.ISSUER);
String[] pubkeys = profilesByPubkeys.keySet().toArray(new String[0]);
int fetchSize = 100;
for (int i = 0; i < pubkeys.length; i += fetchSize) {
//log.debug("Check user profile's settings' {}/{}...", i, pubkeys.length);
int end = Math.min(pubkeys.length, i+fetchSize);
// Get sent movements (where profile is the issuer)
MovementFilter filter = MovementFilter.builder()
.issuers(Arrays.copyOfRange(pubkeys, i, end))
.fields(new String[]{Movement.Fields.ISSUER, Movement.Fields.MEDIAN_TIME})
.build();
service.findMovements(peer, filter, null)
.map(Movement::getIssuer)
// Remove from the result map
.map(profilesByPubkeys::remove)
.filter(Objects::nonNull)
// Add removed profile to the reject list
.forEach(rejectedProfiles::add);
// Get received movements (where profile is the recipient)
filter = MovementFilter.builder()
.recipients(Arrays.copyOfRange(pubkeys, i, end))
.fields(new String[]{Movement.Fields.ISSUER, Movement.Fields.MEDIAN_TIME})
.build();
service.findMovements(peer, filter, null)
.map(Movement::getIssuer)
// Remove from the result map
.map(profilesByPubkeys::remove)
.filter(Objects::nonNull)
// Add removed profile to the reject list
.forEach(rejectedProfiles::add);
}
List<UserProfile> profilesNoTx = Lists.newArrayList(profilesByPubkeys.values());
int delta = profiles.size() - profilesNoTx.size();
log.info("Found {} profiles without Tx ({} has Tx}", profilesNoTx.size(), delta);
return profilesNoTx;
}
protected void deleteProfiles(List<UserProfile> profiles) {
if (CollectionUtils.isEmpty(profiles)) return; // Skip
long now = System.currentTimeMillis();
log.error("Deleting {} fake profiles...", profiles.size());
int counter = 0;
while(counter < profiles.size()) {
UserProfile profile = profiles.get(counter);
try {
try {
service.delete(peer, wallet, profile);
counter++;
if (counter % 50 == 0) {
log.info("Deleted {}/{} ...", counter, profiles.size());
Thread.sleep(30000); // Waiting POD to process
}
}
catch (HttpTimeoutException e) {
// Wait 10s, then loop
Thread.sleep(30000);
}
catch (HttpNotFoundException nfe) {
// OK, continue. Document has been deleted, during a timeout...
counter++;
}
}
catch (HttpUnauthorizeException e) {
throw new TechnicalException("Not authorized to delete a profile. Please check you are using a admin wallet. See test config file", e);
}
catch (Exception e) {
log.error("Cannot delete profile {{}}: {}", profile.getIssuer(), e.getMessage());
throw new TechnicalException("Failed to delete profile " + profile.getIssuer(), e);
}
}
log.info("All profiles deleted, in {}ms", System.currentTimeMillis() - now);
}
protected List<UserProfile> loadProfiles(@NonNull UserProfileFilter filter) {
return loadProfiles(filter, UserProfile.Fields.ISSUER, null);
}
protected List<UserProfile> loadProfiles(@NonNull UserProfileFilter filter, String sortBy, SortDirection sortDirection) {
//Preconditions.checkNotNull(filter.getStartDate());
//Preconditions.checkNotNull(filter.getEndDate());
if (ArrayUtils.isEmpty(filter.getFields())) {
filter.setFields(
new String[]{UserProfile.Fields.ISSUER,
UserProfile.Fields.TITLE,
UserProfile.Fields.DESCRIPTION,
UserProfile.Fields.TIME,
UserProfile.Fields.ADDRESS,
UserProfile.Fields.CITY,
UserProfile.Fields.AVATAR + "." + Attachment.JsonFields.CONTENT_TYPE,
UserProfile.Fields.GEO_POINT
});
}
List<UserProfile> profiles = Lists.newArrayList();
Page page = Page.builder()
.size(1000)
.sortBy(sortBy)
.sortDirection(sortDirection)
.build();
int pageNumber = 1;
try {
boolean hasMorePage;
do {
List<UserProfile> pageResult = service.findAllByFilter(peer, filter, page)
.collect(Collectors.toList());
int resultCount = CollectionUtils.size(pageResult);
if (resultCount > 0) profiles.addAll(pageResult);
hasMorePage = resultCount == page.getSize();
page.setFrom(pageNumber * page.getSize());
pageNumber++;
} while (hasMorePage);
}
catch (Exception e) {
log.error("Error while fetching page #{}: {}", pageNumber, e.getMessage());
// Continue
}
log.info("{} profiles loaded, from filter", profiles.size());
return profiles;
}
private double round(double value, int nbDecimals) {
return ((double)Math.round(value * Math.pow(10, nbDecimals) -0.5)) / Math.pow(10, nbDecimals);
}
private <P, V> List<UserProfile> analyseAndFilterOnField(List<UserProfile> profiles,
List<UserProfile> rejectedProfiles,
String propertyPath,
Function<P, V> propertyValueReducer,
double filterThresholdPct) {
return analyseAndFilterOnField(profiles, rejectedProfiles,
propertyPath,
(p) -> Beans.getProperty(p, propertyPath),
propertyValueReducer,
filterThresholdPct);
}
private <P, V> List<UserProfile> analyseAndFilterOnField(List<UserProfile> profiles,
List<UserProfile> rejectedProfiles,
String ruleName,
Function<UserProfile, P> propertyGetter,
Function<P, V> propertyValueReducer,
double filterThresholdPct) {
Map<V, Integer> counterMap = Maps.newHashMap();
profiles.forEach(profile -> {
P property = propertyGetter.apply(profile);
V value = propertyValueReducer.apply(property);
if (value != null) {
Integer counter = counterMap.get(value);
counterMap.put(value, (counter != null ? counter : 0) + 1);
}
});
Comparator<Map.Entry<V, Integer>> valueComparator = Comparator.comparing(Map.Entry::getValue);
Stream<Map.Entry<V, Integer>> stream = counterMap.entrySet().stream();
// Sorte by nb profiles (desc)
if (!ruleName.toLowerCase().contains("time")) {
stream = stream.sorted(valueComparator.reversed());
}
else {
stream = stream.sorted((e1, e2) -> {
String time1 = (String)e1.getKey();
String time2 = (String)e2.getKey();
return time1.compareTo(time2);
});
}
// Display value / nb profiles
stream
.map(Map.Entry::getKey)
.forEach(value -> {
int counter = counterMap.get(value);
log.trace("{}={} nbProfiles={}", ruleName, value, counter);
});
// Get the major value
V majorValue = counterMap.entrySet().stream()
.sorted(valueComparator.reversed())
.map(Map.Entry::getKey)
.findFirst().orElse(null);
if (counterMap.size() == 0) {
log.debug("No value found for {} [SKIP]", ruleName);
}
else if (counterMap.size() == 1) {
log.debug("All profiles have {}={} [OK]", ruleName, majorValue);
}
else {
// Check of major > threshold
int count = counterMap.entrySet().stream()
.sorted(valueComparator)
.map(Map.Entry::getValue).reduce(0, Integer::sum);
double valuePct = round(((double)counterMap.get(majorValue)) / count * 100, 2);
if (valuePct >= filterThresholdPct) {
log.debug("{}% profiles have {}={} => [FILTERING]", valuePct, ruleName, majorValue);
return profiles.stream().filter(profile -> {
P property = propertyGetter.apply(profile);
V value = propertyValueReducer.apply(property);
if (!Objects.equals(value, majorValue)) {
rejectedProfiles.add(profile);
return false;
}
return true;
}).collect(Collectors.toList());
}
else {
log.debug("{}% profiles have {}={} [SKIP] (thresholdPct={}%)", valuePct, ruleName, majorValue, filterThresholdPct);
}
}
return profiles;
}
protected void dumpToGeoJsonFile(
@NonNull List<UserProfile> profiles,
@NonNull File file,
boolean deleteIfExists) throws IOException {
ObjectMapper objectMapper = JacksonUtils.getThreadObjectMapper();
// Write fake profiles into GeoJson file
if (file.exists()) {
if (deleteIfExists) FileUtils.delete(file);
else throw new TechnicalException("File already exists");
}
List<UserProfile> geoProfiles = profiles.stream().filter(this::hasGeoPoint).collect(Collectors.toList());
if (geoProfiles.size() > 0) {
objectMapper.writeValue(file, service.toGeoJson(geoProfiles));
log.debug("Write {} Geo profiles at {}", geoProfiles.size(), file.getAbsolutePath());
}
}
protected void dumpToFile(
@NonNull List<UserProfile> profiles,
@NonNull File file,
boolean deleteIfExists) throws IOException {
ObjectMapper objectMapper = JacksonUtils.getThreadObjectMapper();
// Write fake profiles into GeoJson file
if (file.exists()) {
if (deleteIfExists) FileUtils.delete(file);
else throw new TechnicalException("File already exists");
}
if (profiles.size() > 0) {
objectMapper.writeValue(file, profiles);
log.debug("Write {} profiles at {}", profiles.size(), file.getAbsolutePath());
}
}
protected List<UserProfile> loadProfilesFile(@NonNull File jsonFile) throws IOException {
Assert.assertTrue(String.format("File %s not exists", jsonFile.getAbsolutePath()), jsonFile.exists());
ObjectMapper objectMapper = JacksonUtils.getThreadObjectMapper();
ArrayNode jsonArray = objectMapper.readValue(Files.newInputStream(jsonFile.toPath()), ArrayNode.class);
Assert.assertNotNull(jsonArray);
final List<UserProfile> result = Lists.newArrayList();
jsonArray.forEach(node -> {
UserProfile profile = null;
try {
profile = objectMapper.treeToValue(node, UserProfile.class);
} catch (JsonProcessingException e) {
log.error("Cannot deserialize a profile: " + node.asText(), e);
}
result.add(profile);
});
log.info("Loaded {} profiles from file {}", result.size(), jsonFile.getAbsolutePath());
return result;
}
protected Set<String> createDictionary(List<UserProfile> profiles) {
return createDictionary(profiles, UserProfile::getTitle);
}
protected Set<String> createDictionary(List<UserProfile> profiles, Function<UserProfile, String> getter) {
Set<String> dictionary = profiles.stream()
.map(getter)
.flatMap(this::streamWords)
.filter(word -> word.length() > 2)
.sorted()
.collect(Collectors.toCollection(LinkedHashSet::new));
log.info("Find {} words, from {} profiles", dictionary.size(), profiles.size());
Assume.assumeTrue(CollectionUtils.isNotEmpty(dictionary));
return dictionary;
}
protected boolean hasGeoPoint(UserProfile p) {
return p.getGeoPoint() != null && p.getGeoPoint().getLat() != null && p.getGeoPoint().getLon() != null;
}
private Pattern postalCodeRegxExp = Pattern.compile("[0-9]{1,5}");
protected boolean hasPostalCode(String city) {
return StringUtils.isNotBlank(city) && postalCodeRegxExp.matcher(city).find();
}
protected List<UserProfile> filterDictionary(List<UserProfile> profiles,
Set<String> dictionary) {
return filterDictionary(profiles, dictionary, UserProfile::getTitle);
}
protected List<UserProfile> filterDictionary(List<UserProfile> profiles,
Set<String> dictionary,
Function<UserProfile, String> getter) {
return profiles.stream()
.filter(p -> streamWords(getter.apply(p))
.anyMatch(dictionary::contains))
.collect(Collectors.toList());
}
protected Stream<String> streamWords(String phrase) {
return (phrase == null)
? Stream.empty()
: Arrays.stream(phrase.split("[ \t]+"))
.map(String::trim)
.map(String::toLowerCase);
}
protected int computeProfileHashCode(List<UserProfile> profiles) {
return profiles.stream()
.map(UserProfile::getIssuer)
.sorted()
.collect(Collectors.toCollection(LinkedHashSet::new))
.hashCode();
}
}
package org.duniter.cesium.pod.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.duniter.cesium.pod.TestResource;
import org.duniter.core.client.model.bma.jackson.JacksonUtils;
import org.duniter.core.exception.TechnicalException;
import org.duniter.core.util.ArrayUtils;
import org.duniter.core.util.Beans;
import org.duniter.core.util.CollectionUtils;
import org.duniter.elasticsearch.model.Page;
import org.duniter.elasticsearch.model.SortDirection;
import org.duniter.elasticsearch.client.model.filter.MovementFilter;
import org.duniter.elasticsearch.client.model.filter.UserProfileFilter;
import org.duniter.elasticsearch.client.model.geom.Envelope;
import org.duniter.elasticsearch.client.service.UserProfileService;
import org.duniter.elasticsearch.model.blockchain.Movement;
import org.duniter.elasticsearch.model.type.Attachment;
import org.duniter.elasticsearch.model.user.UserProfile;
import org.geojson.FeatureCollection;
import org.junit.*;
import java.io.File;
import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;
@Slf4j
public class UserProfileServiceTest extends AbstractServiceTest<UserProfileService>{
@ClassRule
public static final TestResource resource = TestResource.create();
public UserProfileServiceTest(){
super(UserProfileService.class);
}
@Before
public void setUp() {
super.setUp();
peer = getPeer(resource.getFixtures().getDefaultCurrency());
wallet = getWallet(resource.getFixtures().getDefaultCurrency(),
null,
resource.getFixtures().getUserPublicKey(),
resource.getFixtures().getUserSecretKey()
);
}
@Test
public void findAllByFilter() {
// Should match some profiles (bbox France)
{
UserProfileFilter filter = UserProfileFilter.builder()
.boundingBox(Envelope.builder()
.minX(-10).maxX(12)
.minY(40).maxY(54)
.build())
.queryString("Lavenier")
.build();
List<UserProfile> profiles = service.findAllByFilter(peer, filter, null)
.peek(up -> log.info(" - {}", up.getTitle()))
.collect(Collectors.toList());
Assert.assertNotNull(profiles);
Assert.assertTrue(profiles.size() > 0);
}
// Should not match (bbox USA)
{
UserProfileFilter filter = UserProfileFilter.builder()
.boundingBox(Envelope.builder()
.minX(-74).minY(40)
.maxX(-72).maxY(42)
.build())
.queryString("Lavenier")
.build();
List<UserProfile> profiles = service.findAllByFilter(peer, filter, null)
.peek(up -> log.info(" - {}", up.getTitle()))
.collect(Collectors.toList());
Assert.assertNotNull(profiles);
Assert.assertTrue(profiles.isEmpty());
}
// Get only some fields
{
UserProfileFilter filter = UserProfileFilter.builder()
.queryString("Lavenier")
.fields(new String[]{UserProfile.Fields.TITLE})
.build();
List<UserProfile> profiles = service.findAllByFilter(peer, filter, null)
.peek(up -> log.info(" - {}", up.getTitle()))
.collect(Collectors.toList());
Assert.assertNotNull(profiles);
Assert.assertTrue(profiles.size() > 0);
}
}
@Test
public void findMovements() {
MovementFilter filter = MovementFilter.builder()
.pubkey(resource.getFixtures().getUserPublicKey())
.build();
Page page = Page.builder()
.size(10)
.build();
List<Movement> movements = service.findMovements(peer, filter, page)
.collect(Collectors.toList());
Assert.assertNotNull(movements);
Assert.assertTrue(movements.size() > 0);
}
@Test
public void toGeoJson() {
// Should match some profiles (bbox France)
{
UserProfileFilter filter = UserProfileFilter.builder()
.boundingBox(Envelope.builder()
.minX(-10).maxX(12)
.minY(40).maxY(54)
.build())
.queryString("Lavenier")
.build();
List<UserProfile> profiles = service.findAllByFilter(peer, filter, null)
.collect(Collectors.toList());
Assume.assumeNotNull(profiles);
Assume.assumeTrue(profiles.size() > 0);
FeatureCollection geoJson = service.toGeoJson(profiles);
Assert.assertNotNull(geoJson);
Assert.assertTrue(CollectionUtils.isNotEmpty(geoJson.getFeatures()));
Beans.getStream(geoJson.getFeatures()).forEach(feature -> {
Assert.assertNotNull(feature);
Assert.assertNotNull(feature.getGeometry());
Assert.assertNotNull(feature.getProperties());
Assert.assertTrue(feature.getProperties().containsKey(UserProfile.Fields.ISSUER));
Assert.assertTrue(feature.getProperties().containsKey(UserProfile.Fields.TITLE));
});
// Dump to file
try {
File output = new File(resource.getResourceDirectory(), "profiles.geojson");
ObjectMapper objectMapper = JacksonUtils.getThreadObjectMapper();
objectMapper.writeValue(output, geoJson);
log.info("GeoJSon serialize into " + output.getAbsolutePath());
FileUtils.copyFile(output, new File("target", output.getName()));
}
catch (IOException e) {
Assert.fail("Cannot serialize profiles as geoJson: " + e.getMessage());
}
}
}
@Test
public void save() {
// Save
{
UserProfile source = UserProfile.builder()
.title("This is a test profile")
.build();
UserProfile savedProfile = service.save(peer, wallet, source);
Assert.assertNotNull(savedProfile);
Assert.assertEquals(savedProfile.getIssuer(), wallet.getPubKeyHash());
Assert.assertEquals(savedProfile.getIssuer(), savedProfile.getId());
}
// Update
{
UserProfile source = UserProfile.builder()
.title("Another test profile")
.build();
UserProfile savedProfile = service.update(peer, wallet, source);
Assert.assertNotNull(savedProfile);
Assert.assertEquals(savedProfile.getIssuer(), wallet.getPubKeyHash());
Assert.assertEquals(savedProfile.getIssuer(), savedProfile.getId());
}
}
@Test
public void createProfilesMap() throws IOException {
long now = System.currentTimeMillis();
List<UserProfile> profiles = loadAllProfilesWithGeoPoint(
UserProfile.Fields.ISSUER,
UserProfile.Fields.TITLE,
UserProfile.Fields.CITY,
UserProfile.Fields.GEO_POINT);
Assert.assertNotNull(profiles);
Assert.assertFalse(profiles.isEmpty());
log.info("Loaded {} profiles in {}ms", profiles.size(), System.currentTimeMillis() - now);
File output = new File("target/profiles-map.geojson");
dumpToGeoJsonFile(profiles, output, true);
}
/* -- protected -- */
protected List<UserProfile> loadAllProfilesWithGeoPoint(String... fields) {
long now = System.currentTimeMillis();
List<UserProfile> result = Lists.newArrayList();
int latDelta = 10;
int lonDelta = 20;
int counter = 0;
for (int lat = -90; lat < 90; lat += latDelta) {
for (int lon = -180; lon < 180; lon += lonDelta) {
counter++;
if (counter % 10 == 0) log.trace("Loading geo slice #{}...", counter);
UserProfileFilter filter = UserProfileFilter.builder()
.boundingBox(Envelope.builder()
.minX(lon).maxX(lon+lonDelta)
.minY(lat).maxY(lat+latDelta)
.build())
.fields(fields)
.build();
List<UserProfile> slice = loadProfiles(filter);
result.addAll(slice);
}
}
log.debug("Loaded {} slices loaded in {}ms", counter, System.currentTimeMillis() - now);
return result;
}
protected List<UserProfile> loadProfiles(@NonNull UserProfileFilter filter) {
return loadProfiles(filter, UserProfile.Fields.ISSUER, null);
}
protected List<UserProfile> loadProfiles(@NonNull UserProfileFilter filter, String sortBy, SortDirection sortDirection) {
if (ArrayUtils.isEmpty(filter.getFields())) {
filter.setFields(
new String[]{UserProfile.Fields.ISSUER,
UserProfile.Fields.TITLE,
UserProfile.Fields.DESCRIPTION,
UserProfile.Fields.TIME,
UserProfile.Fields.ADDRESS,
UserProfile.Fields.CITY,
UserProfile.Fields.AVATAR + "." + Attachment.JsonFields.CONTENT_TYPE,
UserProfile.Fields.GEO_POINT
});
}
List<UserProfile> profiles = Lists.newArrayList();
Page page = Page.builder()
.size(1000)
.sortBy(sortBy)
.sortDirection(sortDirection)
.build();
int pageNumber = 0;
try {
boolean hasMorePage;
do {
page.setFrom(pageNumber++ * page.getSize());
List<UserProfile> pageResult = service.findAllByFilter(peer, filter, page)
.collect(Collectors.toList());
int resultCount = CollectionUtils.size(pageResult);
if (resultCount > 0) profiles.addAll(pageResult);
hasMorePage = resultCount == page.getSize();
} while (hasMorePage);
}
catch (Exception e) {
log.error("Error while fetching page #{}: {}", pageNumber, e.getMessage());
// Continue
}
log.trace("{} profiles loaded ({} pages)", profiles.size());
return profiles;
}
protected void dumpToGeoJsonFile(
@NonNull List<UserProfile> profiles,
@NonNull File file,
boolean deleteIfExists) throws IOException {
ObjectMapper objectMapper = JacksonUtils.getThreadObjectMapper();
// Write fake profiles into GeoJson file
if (file.exists()) {
if (deleteIfExists) FileUtils.delete(file);
else throw new TechnicalException("File already exists");
}
// Keep only profile with a point
List<UserProfile> geoProfiles = profiles.stream()
.filter(this::hasGeoPoint)
.collect(Collectors.toList());
if (geoProfiles.size() > 0) {
objectMapper.writeValue(file, service.toGeoJson(geoProfiles));
log.info("Write {} Geo profiles at {}", geoProfiles.size(), file.getAbsolutePath());
}
}
protected boolean hasGeoPoint(UserProfile p) {
return p.getGeoPoint() != null && p.getGeoPoint().getLat() != null && p.getGeoPoint().getLon() != null;
}
}
package org.duniter.cesium.pod.service;
/*-
* #%L
* Cesium+ pod :: Client API
* %%
* Copyright (C) 2014 - 2023 Duniter Team
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
import lombok.extern.slf4j.Slf4j;
import org.duniter.cesium.pod.TestResource;
import org.duniter.elasticsearch.client.model.filter.UserSettingsFilter;
import org.duniter.elasticsearch.client.service.UserSettingsService;
import org.duniter.elasticsearch.model.user.UserSettings;
import org.junit.*;
import java.util.List;
import java.util.stream.Collectors;
@Slf4j
public class UserSettingsServiceTest extends AbstractServiceTest<UserSettingsService>{
@ClassRule
public static final TestResource resource = TestResource.create();
public UserSettingsServiceTest(){
super(UserSettingsService.class);
}
@Before
public void setUp() {
super.setUp();
peer = getPeer(resource.getFixtures().getDefaultCurrency());
wallet = getWallet(resource.getFixtures().getDefaultCurrency(),
null,
resource.getFixtures().getUserPublicKey(),
resource.getFixtures().getUserSecretKey()
);
}
@Test
public void findAllByFilter() {
// Should match some profiles (bbox France)
{
UserSettingsFilter filter = UserSettingsFilter.builder()
.issuers(new String[]{
wallet.getPubKeyHash()
})
.build();
List<UserSettings> profiles = service.findAllByFilter(peer, filter, null)
.peek(up -> log.debug(" - issuer: {}", up.getIssuer()))
.collect(Collectors.toList());
Assert.assertNotNull(profiles);
Assert.assertTrue(profiles.size() > 0);
}
// Get only some fields
{
UserSettingsFilter filter = UserSettingsFilter.builder()
.issuers(new String[]{
wallet.getPubKeyHash()
})
.fields(new String[]{UserSettings.Fields.ISSUER, UserSettings.Fields.TIME})
.build();
List<UserSettings> profiles = service.findAllByFilter(peer, filter, null)
.peek(up -> log.debug(" - time: {}", up.getTime()))
.collect(Collectors.toList());
Assert.assertNotNull(profiles);
Assert.assertTrue(profiles.size() > 0);
}
}
@Test
public void findByPubkey() {
UserSettings settings = service.findByPubkey(peer, resource.getFixtures().getUserPublicKey()).orElseGet(null);
Assert.assertNotNull(settings);
Assert.assertNotNull(settings.getIssuer());
Assert.assertNotNull(settings.getContent());
Assert.assertNotNull(settings.getTime());
Assert.assertNotNull(settings.getNonce());
}
@Test
@Ignore
public void save() {
// TODO
UserSettings source = UserSettings.builder()
//.content()
.build();
UserSettings savedSettings = service.save(peer, wallet, source);
Assert.assertNotNull(savedSettings);
Assert.assertEquals(savedSettings.getIssuer(), wallet.getPubKeyHash());
Assert.assertEquals(savedSettings.getIssuer(), savedSettings.getId());
}
}
#duniter4j.node.host=g1.data.e-is.pro
#duniter4j.node.port=443
#duniter4j.node.host=192.168.0.45
#duniter4j.node.port=9200
#duniter4j.node.host=data.gchange.fr
#duniter4j.node.port=443
#duniter4j.node.host=g1s.data.e-is.pro
#duniter4j.node.port=80
duniter4j.node.host=g1.data.e-is.pro
duniter4j.node.port=443
\ No newline at end of file
ovale, trois, Rat, Lamarckie, ailé, d’Europe, Macaque, Zircon, Gayane, bifascié, Iris, Hutia, Lynx, Pierrine, multiflore, Harpie, Reuel, Platanthère, Trèfle, écureuil, Cirse, Gagée, Epilobe, Anémone, Figuier, Cardinal, Heimana, aphaca, Cytise, pied-d'oiseau, Carcajou, Corydale, arc-en-ciel, chromifère, printemps, impériale, Orpin, marron, Pisaure, d'Ortie, Açores, Ginkgo, Mouron, Plectrophane, marginée, minuscule, Crocus, l'Est, Beaux, Morgan, hirsute, Phytolacca, multicolore, Orphée, Veuve, Gesse, Billard, Lamproie, Noredine, ronde, Sabline, Vannie, Rhinopithèque, feuilles, Néotine, coeur, Belle, Grande-gueule, Bignone, Vipérine, Passerage, Oeillet, Caracara, Casseille, Carotte, Sybel, Cordiérite, grosses, Drave, pervenche, Eudora, Galles, épineux, Bugrane, Papillon, Airelle, Pissenlit, marais, épi, Fourmi, européen, Bilail, Huîtrier-pie, Selvi, étoilé, faux, Ratufa, Héliolite, Béryl, tête-de-boeuf, Charançon, noire, Séneçon, oroméditerranéen, Lobelia, Pays, Violette, papillon, Nawel, Chicorée, Merle, à, Obsidienne, Nautile, linaire, Vive, Nerprun, Hibiscus, Salsifis, Kiwi, violette, Chèvrefeuille, biloba, collier, Krispin, vinaigre, Hong, printanière, sapins, Madère, Ficoïde, Cosentinia, Eglantier, Sterne, Cigale, Ylang-ylang, Johannie, Virgilier, Ondatra, alkekenge, Camélia, Angwantibo, Rhinanthe, Grenouille, Ail, Kalanchoë, Zinnia, précoce, Virginie, Margie, Abgar, Gaur, Oseille, Elberta, Manul, Muflier, Libocèdre, Séquoia, Grillon, d'Autriche, Furet, Onopordon, Longanier, Véronique, Faucon, Cornouiller, Araignée, Fuchs, larges, Bonobo, Merlebleu, malaya, Andésine, fasciée, chevelu, trilobé, Microcline, Diamant, Ringhal, opposées, Pic, Pie, tubéreuse, Pin, Mouche, Sphinx, rose, ailes, Pyrole, Barbeau, Glaïeul, semidécurente, Accenteur, Shina, alba, Sénevé, Cenis, penché, Baliste, Ambre, Listère, points, Androsace, Orchis, roux, dorée, Madrone, Scarabée, Crevette, Varan, Stellaire, Ly, d'Espagne, arborescente, Nelly, Grenat, odorante, Oeil, Tourmaline, Colibri, Ancolie, Bianka, Gouet, Frangipanier, Daphné, Kanoa, Baleine, grièche, buse, Astérolide, Brugmansia, Verdier, couchée, Japon, Écureuil, Phaner, murs, Citrine, Ruby, Demi, Herbe, Raiponce, alpestre, brillant, Argyrolobe, sarde, Carline, Napoléon, Smaragdite, Reuter, Doronic, Calcite, Iguane, Achillée, Marguerite, léopard, Linaigrette, Arbousier, Brachychiton, Caroubier, Lotier, Rhésus, Pied-de-poule, Parnassie, Mélèze, grandes, Seringat, Aurora, Gwendolina, cordées, prés, Wapiti, bois, Primevère, rameuse, Yèble, Aconit, cantabrique, Laurier, nobilis, renflé, Paronyque, Siamang, Jonquille, Arisarum, Loir, Muscari, nain, Labbe, Ara, Gambas, Logan, barbu, Pou, Coyote, Crystel, Ioanna, Pulmonaire, marine, Cétoine, grise, Mélisse, luisant, Tareja, fruits, Hippopotame, Frelon, Puma, réfléchi, Kodkod, colier, Martin, Quartz, argentée, Lin, fausse-phlomide, Lis, Marouette, Lychnis, Clorinda, d'Italie, tuberculé, Lilas, Gentiane, Céphalanthère, Misopate, Aéthionéma, Lingue, Crabe, yeux, Perruche, Tanaisie, Alchémille, Jean-le-Blanc, Goundi, Férule, lapon, Woma, ramifié, sauvage, Adonis, Requin, Baguenaudier, Kenny, Sérapias, petit-chêne, épaisses, Potentille, quatre, Carouge, maritime, crête-de-Coq, Greten, Amétrine, Andradite, aux, commune, Pacarana, Mara, Micocoulier, Couscous, Mainate, Aiguillat, Julienne, Ilhem, noires, graines, zoïsite, pain, Lavande, Larinioides, Asphodèle, Galinette, Calendula, Pensée, Capricorne, tropicale, loups, fleur, Bécasse, jardins, Harun, Sandre, Sylvestine, jaunâtre, Zanon, Mangabey, conifère, Scille, Marigane, crocus, Marilyne, élégant, petites, pacifique, Pallénis, démantoïde, Xatardie, nervosa, violet, Laitue, Nouhaïla, Lieu, Constentin, Meddy, Saltique, Alpes, typographe, Fahd, Punaise, perles, Kravat, Ouananiche, Millepertuis, Sardoine, Lysimaque, Calament, Sarriette, Thon, Agrion, Nice, plateau, Sanah, Oursin, Loup, noirâtre, Rosier, d'acanthe, Patas, boréal, Narval, Hortensia, gris, Grande, méconnu, japonais, Gobe-mouche, chat, mouchet, Catananche, Bénito, Liane, Canard, Soude, Jadéite, Calao, Prosopis, Grémil, écarlate, Nawell, Actinote, loutre, articulé, Erable, feu, souchet, Fouhad, Coronille, Janney, Scheuchzer, arbustive, cravant, volant, Nombril, judé, Saphir, Requin-baleine, d'Adonis, officinal, Crabier, rampante, rhinocéros, Pois, grecque, bouc, alpine, Carillon, montagne, Poisson-zèbre, très, Jasione, polyglotte, acinos, Matthiole, Zorille, petits, Carabe, Séguier, Dompte-venin, Rainette, Rollande, Cardère, Zygopetalum, Mauve, Butor, Barge, Cygne, montagnes, Genêt, Spinelle, Arbre, terrier, Hellébore, Cardamine, Durieu, sancta, Olivier, Orcanette, Blennie, Orobanche, Céraste, Marjolaine, sables, Goéland, teinturiers, Anophèle, Pyrénées, Hélianthème, Xéranthème, Micromeria, des, Girafe, Marwan, Tadorne, d’Hermann, siffleur, Lycium, Poisson-pilote, Physalis, Ayoub, Émeu, solstice, Corégone, Anne-Lyse, Panthère, Californie, rayée, Pétrel, Ouistiti, chêne, Jade, Titan, Ciste, Tarentule, velue, Narcisse, Bibacier, Vigne, fuligule, migratrice, Margouillat, Abricotier, Scabieuse, Pékan, Sicile, Vélar, Capret, Zabre, Plantain, Renoncule, Cinglard, Cistanche, Râle, Cicadelle, Randy, Leuzée, Raïhane, Sisymbre, pimprenelle, digitée, Luzerne, Bananier, pygmée, noueux, hybride, Queue-de-lièvre, rondes, Isabella, berbère, Phoque, de, acerifolius, Courlan, bec, Muge, Férid, Astragale, pourpre, Thym, Grémille, du, languettes, Léopard, fleurs, Congre, Bécassin, Fuligule, Coccinelle, Emmanuella, céréales, Zenobia, Vandoise, odorant, pêcheur, en, Quokka, doré, Scapolite, Suricate, Corrine, Nyssa, casarca, Darin, Mélampyre, bleue, Barba, Melvin, Molène, Amazone, Tortue, Liseron, Bec-croisé, Kolkwitzia, Nandou, Linotte, anémones, Chat-pêcheur, Chevêche, tigre, Phidippus, Ononis, comosum, blanc, Lisette, lierre, Sainfoin, une, Polygala, Quiscale, Alliaire, Grand, Smithsonite, Héron, Jehenos, laineuse, Genna, cendré, Erodium, Karianna, officinale, moissons, vert, Bruyère, Xylocope, militaire, Geoffroy, Pouillot, Jaoued, Acarien, Bartsie, opium, Andalousite, Bugle, cornes, Quetzal, Amorpha, Hypolaïs, senteur, flammé, Jeremejevite, nébuleuse, Axelane, Provence, Hyacinthe, Guifette, Panda, Rodié, Canada, lune, Succise, désert, fitis, Jaouen, Cynoglosse, Homard, sanguin, Magellan, Torreya, Ailante, Pulsatille, Aster, Chardon, Frégate, Catalogne, Rose, Rosa, Numbat, Lérotin, Géranium, Saxifrage, Bernache, Tofieldie, Turquoise, Méissa, Charlette, Colin, Lamier, Sphynx, listère, Sillimanite, Petite, faux-acanthe, la, Éland, Floréal, Calixa, Epiaire, Cytinet, Mateo, Moqueur, calice, Fushia, Circaète, sauge, hérissé, Oxalis, Lézard, vessie, radis, Crète, rubis, Diopside, labradorite, Jaseur, Monserrat, Giroflée, Moricandie, commun, Mectilde, Mygale, Heather, Scorsonère, Saul, Leblond, Asiminier, Oie, jacée, Caroline, Danaëlle, œil, Yack, Mimule, Lièvre, Pavot, Pédiculaire, Joubarbe, Tsuga, Brème, Eleuteriu, rayé, mako, Servanne, Réséda, Tricot, Euphorbe, Mandarin, mer, Filao, Tamarillo, Aristoloche, Wilson, Cormoran, Lotus, Chardonneret, familier, orange, Weta, duc, rousse, neiges, champêtre, Pietersite, Âne, Lucilie, Wollemia, palmier, Gaillet, maculé, Gamal, delta, Squille, fausse-raiponce, Nénuphar, Vesce, Bostryche, Oryane, Ophrys, Olm, Loubine, Saponaire, filiforme, pointes, Pécaris, tomenteuse, rude, Fatsia, Magot, Vrillette, Oxytropis, aethiopica, Huitre, tacheté, Courbine, Bécasseau, d'Aragon, Plumier, Pacanier, Fluorite, Criquet, bigarrée, Mimosa, Étrille, Lazuli, ramifiée, Troglodyte, bleu, jaune, cactus, Brazilianite, Yucca, nummulaire, Sauge, étroites, Ethérie, Castor, Barytine, Teanuanua, Buffle, chevronnée, Hercule, Cafard, Mélissanne, Pouilles, intermédiaire, Truite, albacore, Bilimbi, Vénus, Cisar, Panga, Goret, Bonelli, Chios, Genève, laineux, Azerolier, Evelina, Avocette, Brunelle, sylvestres, Ciboulette, Exuperi, Painite, noir, Blanchet, Calcédoine, Lisandra, Mélipone, vingt-deux, nonnette, Dendrocygne, admirable, Parisette, Bongare, blanche, Hérisson, Arum, ventre, Pygargue, Calliphoridé, Corail, Clématite, serpolet, Globulaire, Makaire, brun, Grassette, Faisan, maubèche, Maïssane, deux, rouge, Acanthe, Cléo, aggloméré, ciliée, Susan, d'Irlande, vivace, Bruant, verticillée, Cassien, grande, Aubriète, Arthuria, parasite, Tigre, Coris, Camomille, Juan, dents, Petit, vulgaire, Dorine, Mélilot, ensanglantée, penchée, Aralia, Aucuba, d'Amérique, mou, pleureur, Méconopsis, Jacaranda, Échidné, chevalier, Escargot, indica, Englebert, Danburite, Chrysanthèmes, Ortie, Soazick, Sardonyx, Hamamélis, Anguille, bouillon, Lupin, collines, argenté, Mont, Tétraodon, longues, Anthyllide, Montpellier, Doumenge, Launée, fausse, gorge, pèlerin, Notoceras, Germandrée, asiatique, Centaurée, Saule, Vipère, Orchidée, Pierre, Préhnite, Agate, persicaire, Jacobine, Odénie, Zazie, Crapaudine, Abeille, rochers, Piranga, géant, Singe, aquatique, Chat, Gypaète, Nacre, pâle, Mangoustanier, Arabette, Sureau, Mehonia, voyageuse, Patelle, Roselin, Carassin, Dactyle, Phacélie, Rubis, Opale, Euphorbe-cierge, naine, Achod, Engoulevent, Fusain, Lorrain, queue, Alejandro, Limonium, pyrope, Condors, Campanule, Chevalier, Linaire, Ouakari, laciniée, Martinet, Acajou, Putois, Perle, Atlas, Salamandre, imbriqué, Renouée, Gustav, Kapel, Wallace, Bécassine, ibérique, Poliste, Silène, Valériane, Blobfish, Lapis, Apatite, Limace, Serpent, noircissant, champs, Swertie, diable, Datolite, Oiseau, Chien, faux-aizoon, Ivor, Chaï, Menthe, calicule, Elaeagnus, Madyson, Bittor, Zantedeschia, élégante, Sureau-Sambucus, Pétauriste, cœur, tête, Pistachier, sept, Vulcain, verte, Jararaca, Jacinthe
\ No newline at end of file