From 44a4c8a949d1d3783f1fb15090b2f7d93a6337f1 Mon Sep 17 00:00:00 2001
From: Moul <moul@moul.re>
Date: Mon, 24 May 2021 16:38:17 +0200
Subject: [PATCH] [feat] #256: Introduce excluded command for DeathReaper
Introduce some tests: TODO write all tests
---
silkaj/cli.py | 2 +
silkaj/constants.py | 5 +
silkaj/excluded.py | 265 +++++++++++++++++++++++++++++++++++++++++
tests/test_excluded.py | 134 +++++++++++++++++++++
4 files changed, 406 insertions(+)
create mode 100644 silkaj/excluded.py
create mode 100644 tests/test_excluded.py
diff --git a/silkaj/cli.py b/silkaj/cli.py
index b4c633d8..ab48418f 100644
--- a/silkaj/cli.py
+++ b/silkaj/cli.py
@@ -28,6 +28,7 @@ from silkaj.constants import (
G1_TEST_DEFAULT_ENDPOINT,
SILKAJ_VERSION,
)
+from silkaj.excluded import excluded
from silkaj.license import license_command
from silkaj.membership import send_membership
from silkaj.money import cmd_amount
@@ -124,6 +125,7 @@ cli.add_command(list_blocks)
cli.add_command(send_certification)
cli.add_command(checksum_command)
cli.add_command(difficulties)
+cli.add_command(excluded)
cli.add_command(transaction_history)
cli.add_command(id_pubkey_correspondence)
cli.add_command(currency_info)
diff --git a/silkaj/constants.py b/silkaj/constants.py
index cbaee255..a93e60ab 100644
--- a/silkaj/constants.py
+++ b/silkaj/constants.py
@@ -18,12 +18,17 @@ G1_SYMBOL = "Äž1"
GTEST_SYMBOL = "ÄžTest"
G1_DEFAULT_ENDPOINT = "BMAS g1.duniter.org 443"
G1_TEST_DEFAULT_ENDPOINT = "BMAS g1-test.duniter.org 443"
+
+ONE_HOUR = 3600
+
SUCCESS_EXIT_STATUS = 0
FAILURE_EXIT_STATUS = 1
+
BMA_MAX_BLOCKS_CHUNK_SIZE = 5000
PUBKEY_MIN_LENGTH = 43
PUBKEY_MAX_LENGTH = 44
PUBKEY_PATTERN = f"[1-9A-HJ-NP-Za-km-z]{{{PUBKEY_MIN_LENGTH},{PUBKEY_MAX_LENGTH}}}"
+
MINIMAL_ABSOLUTE_TX_AMOUNT = 0.01
MINIMAL_RELATIVE_TX_AMOUNT = 1e-6
CENT_MULT_TO_UNIT = 100
diff --git a/silkaj/excluded.py b/silkaj/excluded.py
new file mode 100644
index 00000000..ed5cc99d
--- /dev/null
+++ b/silkaj/excluded.py
@@ -0,0 +1,265 @@
+# Copyright 2016-2021 Maël Azimi <m.a@moul.re>
+#
+# Silkaj is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Silkaj is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Silkaj. If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import sys
+import time
+import urllib
+
+import click
+import pendulum
+from duniterpy import constants as dp_const
+from duniterpy.api.bma import blockchain
+from duniterpy.api.client import Client
+from duniterpy.documents.block import Block
+from pydiscourse import DiscourseClient
+from pydiscourse.exceptions import DiscourseClientError
+
+from silkaj import constants, wot_tools
+from silkaj.blockchain_tools import BlockchainParams
+from silkaj.network_tools import ClientInstance
+from silkaj.tools import CurrencySymbol
+
+G1_CESIUM_URL = "https://demo.cesium.app/"
+GTEST_CESIUM_URL = "https://g1-test.cesium.app/"
+CESIUM_BLOCK_PATH = "#/app/block/"
+
+DUNITER_FORUM_URL = "https://forum.duniter.org/"
+MONNAIE_LIBRE_FORUM_URL = "https://forum.monnaie-libre.fr/"
+
+DUNITER_FORUM_G1_TOPIC_ID = 4393
+DUNITER_FORUM_GTEST_TOPIC_ID = 6554
+MONNAIE_LIBRE_FORUM_G1_TOPIC_ID = 17627
+
+
+@click.command(
+ "excluded",
+ help="Generate membership exclusions messages, \
+markdown formatted and publish them on Discourse Forums",
+)
+@click.option(
+ "-a",
+ "--api-id",
+ help="Username used on Discourse forum API",
+)
+@click.option(
+ "-du",
+ "--duniter-forum-api-key",
+ help="API key used on Duniter Forum",
+)
+@click.option(
+ "-ml",
+ "--ml-forum-api-key",
+ help="API key used for Monnaie Libre Forum",
+)
+@click.argument("days", default=1, type=click.FloatRange(0, 50))
+@click.option(
+ "--publish",
+ is_flag=True,
+ help="Publish the messages on the forums, otherwise print them",
+)
+def excluded(api_id, duniter_forum_api_key, ml_forum_api_key, days, publish):
+ params = BlockchainParams().params
+ currency = params["currency"]
+ check_options(api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency)
+ bma_client = ClientInstance().client
+ blocks_to_process = get_blocks_to_process(bma_client, days, params)
+ if not blocks_to_process:
+ no_exclusion(days, currency)
+ message = gen_message_over_blocks(bma_client, blocks_to_process, params)
+ if not message:
+ no_exclusion(days, currency)
+ header = gen_header(blocks_to_process)
+ # Add ability to publish just one of the two forum, via a flags?
+
+ publish_display(
+ api_id,
+ duniter_forum_api_key,
+ header + message,
+ publish,
+ currency,
+ "duniter",
+ )
+ if currency == dp_const.G1_CURRENCY_CODENAME:
+ publish_display(
+ api_id,
+ ml_forum_api_key,
+ header + message,
+ publish,
+ currency,
+ "monnaielibre",
+ )
+
+
+def check_options(api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency):
+ if publish and (
+ not api_id
+ or not duniter_forum_api_key
+ or (not ml_forum_api_key and currency != dp_const.G1_TEST_CURRENCY_CODENAME)
+ ):
+ sys.exit(
+ "Error: To be able to publish, api_id, duniter_forum_api, and \
+ml_forum_api_key (not required for {constants.GTEST_SYMBOL}) options should be specified"
+ )
+
+
+def no_exclusion(days, currency):
+ # Humanize
+ sys.exit(f"No exclusion to report within the last {days} day(s) on {currency}")
+
+
+def get_blocks_to_process(bma_client, days, params):
+ head_number = bma_client(blockchain.current)["number"]
+ block_number_days_ago = (
+ head_number - days * 24 * constants.ONE_HOUR / params["avgGenTime"]
+ )
+ # print(block_number_days_ago) # DEBUG
+
+ blocks_with_excluded = bma_client(blockchain.excluded)["result"]["blocks"]
+ for i, block_number in reversed(list(enumerate(blocks_with_excluded))):
+ if block_number < block_number_days_ago:
+ break
+ return blocks_with_excluded[i + 1 :]
+
+
+def gen_message_over_blocks(bma_client, blocks_to_process, params):
+ """
+ Loop over the list of blocks to retrieve and parse the blocks
+ Ignore revocation kind of exclusion
+ """
+ message = ""
+ for block_number in blocks_to_process:
+ logging.info(f"Processing block number {block_number}")
+ print(f"Processing block number {block_number}")
+ # DEBUG / to be removed once the #115 logging system is set
+
+ try:
+ block = bma_client(blockchain.block, block_number)
+ except urllib.error.HTTPError:
+ time.sleep(2)
+ block = bma_client(blockchain.block, block_number)
+ block_hash = block["hash"]
+ block = Block.from_signed_raw(block["raw"] + block["signature"] + "\n")
+
+ if block.revoked and block.excluded[0] == block.revoked[0].pubkey:
+ continue
+ message += await generate_message(block, block_hash, params)
+ return message
+
+
+def gen_header(blocks_to_process):
+ nbr_exclusions = len(blocks_to_process)
+ # Handle when there is one block with multiple exclusion within
+ # And when there is a revocation
+ s = "s" if nbr_exclusions > 1 else ""
+ des_du = "des" if nbr_exclusions > 1 else "du"
+ currency_symbol = CurrencySymbol().symbol
+ header = f"## Exclusion{s} de la toile de confiance {currency_symbol}, perte{s} {des_du} statut{s} de membre"
+ message_g1 = "\n> Message automatique. Merci de notifier vos proches de leur exclusion de la toile de confiance."
+ return header + message_g1
+
+
+def generate_message(block, block_hash, params):
+ """
+ Loop over exclusions within a block
+ Generate identity header + info
+ """
+ message = ""
+ for i, excluded in enumerate(block.excluded):
+ lookup = wot_tools.wot_lookup(excluded)[0]
+ uid = lookup["uids"][0]["uid"]
+
+ if params["currency"] == dp_const.G1_CURRENCY_CODENAME:
+ cesium_url = G1_CESIUM_URL
+ else:
+ cesium_url = GTEST_CESIUM_URL
+ cesium_url += CESIUM_BLOCK_PATH
+ message += f"\n\n### @{uid} [{uid}]({cesium_url}{block.number}/{block_hash}?ssl=true)\n"
+ message += generate_identity_info(lookup, block, params)
+ return message
+
+
+def generate_identity_info(lookup, block, params):
+ info = "- **Certifié·e par**"
+ nbr_different_certifiers = 0
+ for i, certifier in enumerate(lookup["uids"][0]["others"]):
+ if certifier["uids"][0] not in info:
+ nbr_different_certifiers += 1
+ info += elements_inbetween_list(i, lookup["uids"][0]["others"])
+ info += "@" + certifier["uids"][0]
+ if lookup["signed"]:
+ info += ".\n- **A certifié**"
+ for i, certified in enumerate(lookup["signed"]):
+ info += elements_inbetween_list(i, lookup["signed"])
+ info += "@" + certified["uid"]
+ dt = pendulum.from_timestamp(block.mediantime + constants.ONE_HOUR, tz="local")
+ info += ".\n- **Exclu·e le** " + dt.format("LLLL", locale="fr")
+ info += " CET\n- **Raison de l'exclusion** : "
+ if nbr_different_certifiers < params["sigQty"]:
+ info += "manque de certifications"
+ else:
+ info += "expiration du document d'adhésion"
+ # a renouveller tous les ans (variable) humanize(params[""])
+ return info
+
+
+def elements_inbetween_list(i, cert_list):
+ return " " if i == 0 else (" et " if i + 1 == len(cert_list) else ", ")
+
+
+def publish_display(api_id, forum_api_key, message, publish, currency, forum):
+ if publish:
+ topic_id = get_topic_id(currency, forum)
+ publish_message_on_the_forum(api_id, forum_api_key, message, topic_id, forum)
+ elif forum == "duniter":
+ click.echo(message)
+
+
+def get_topic_id(currency, forum):
+ if currency == dp_const.G1_CURRENCY_CODENAME:
+ if forum == "duniter":
+ return DUNITER_FORUM_G1_TOPIC_ID
+ else:
+ return MONNAIE_LIBRE_FORUM_G1_TOPIC_ID
+ else:
+ return DUNITER_FORUM_GTEST_TOPIC_ID
+
+
+def publish_message_on_the_forum(api_id, forum_api_key, message, topic_id, forum):
+ if forum == "duniter":
+ discourse_client = DiscourseClient(
+ DUNITER_FORUM_URL, api_username=api_id, api_key=forum_api_key
+ )
+ else:
+ discourse_client = DiscourseClient(
+ MONNAIE_LIBRE_FORUM_URL,
+ api_username=api_id,
+ api_key=forum_api_key,
+ )
+ try:
+ response = discourse_client.create_post(message, topic_id=topic_id)
+ publication_link(forum, response, topic_id)
+ except DiscourseClientError as error:
+ logging.error(f"Issue publishing on {forum} because: {error}")
+ # Handle DiscourseClient exceptions, pass them to the logger
+
+ # discourse_client.close()
+ # How to close this client? It looks like it is not implemented
+ # May be by closing requests' client
+
+
+def publication_link(forum, response, topic_id):
+ forum_url = DUNITER_FORUM_URL if forum == "duniter" else MONNAIE_LIBRE_FORUM_URL
+ print(f"Published on {forum_url}t/{response['topic_slug']}/{str(topic_id)}/last")
diff --git a/tests/test_excluded.py b/tests/test_excluded.py
new file mode 100644
index 00000000..b44fca27
--- /dev/null
+++ b/tests/test_excluded.py
@@ -0,0 +1,134 @@
+# Copyright 2016-2021 Maël Azimi <m.a@moul.re>
+#
+# Silkaj is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Silkaj is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Silkaj. If not, see <https://www.gnu.org/licenses/>.
+
+import pytest
+from duniterpy.documents.block import Block
+
+from silkaj import excluded
+
+
+@pytest.mark.parametrize(
+ "api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency, error",
+ [
+ (None, None, None, False, "g1", False),
+ (None, None, None, True, "g1", True),
+ ("DeathReaper", None, None, True, "g1", True),
+ (None, "key", None, True, "g1", True),
+ (None, None, "key", True, "g1", True),
+ ("DeathReaper", "key", "key", True, "g1", False),
+ ("DeathReaper", "key", None, True, "g1-test", False),
+ ],
+)
+def test_check_options(
+ api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency, error, capsys
+):
+ if error:
+ with pytest.raises(SystemExit) as pytest_exit:
+ excluded.check_options(
+ api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency
+ )
+ assert pytest_exit.type == SystemExit
+ assert "Error: To be able to publish, api_id" in capsys.readouterr().out
+ else:
+ excluded.check_options(
+ api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency
+ )
+ assert not capsys.readouterr().out
+
+
+"""
+def test_generate_identity_info():
+ ref = "- **Certifié·e par** @GildasMalassinet, @Marcolibri, @Gregory et @gpsqueeek.\n\
+- **A certifié** @lesailesbleues, @Marcolibri, @Gregory et @LaureBLodjtahne.\n\
+- **Exclu·e le** 2020-01-27 04:06:09\n\
+- **Raison de l'exclusion** : manque de certifications"
+ # missing lookup and block
+ # 292110
+ raw_block = "Version: 11\nType: Block\nCurrency: g1\nNumber: 292110\nPoWMin: 89\n\
+Time: 1580098477\nMedianTime: 1580094369\nUnitBase: 0\n\
+Issuer: D3krfq6J9AmfpKnS3gQVYoy7NzGCc61vokteTS8LJ4YH\nIssuersFrame: 161\n\
+IssuersFrameVar: 0\nDifferentIssuersCount: 32\n\
+PreviousHash: 0000009F88010580F54C0403BC87BF10E912D0FBC274294343DDC748F697991D\n\
+PreviousIssuer: 5P9HB1uBdE3ZqThSatBdBbjVmh3psa1BhsCRBumn1aMo\nMembersCount: 2527\n\
+Identities:\nJoiners:\nActives:\nLeavers:\nRevoked:\nExcluded:\n\
+DKWXiEEd4wkf4pgd95vkGnNkHxzTKDpPPSgMay5WJ7Fc\nCertifications:\nTransactions:\n\
+InnerHash: CD17661EBD858CC374918D2F94363E22AB5BDC617FDEEA729C384616FD2285C4\n\
+Nonce: 10800000223569\n"
+ block_sig = "GvREqrzvbS2RHoLCEbfjjPcsJ+G+jVKqejF5fu9i3076XJ4pmc+udyCPoJ5TiCo7OPB5GOLmsCLocJZbqQS1CA=="
+ block = Block.from_signed_raw(raw_block + block_sig + "\n")
+ gen = excluded.generate_identity_info(lookup, block, 5)
+ assert gen == ref
+"""
+
+
+def test_elements_inbetween_list():
+ list = ["toto", "titi", "tata"]
+ assert excluded.elements_inbetween_list(0, list) == " "
+ assert excluded.elements_inbetween_list(1, list) == ", "
+ assert excluded.elements_inbetween_list(2, list) == " et "
+
+
+@pytest.mark.parametrize(
+ "message, publish, currency, forum", [("message", False, "g1", "duniter")]
+)
+def test_publish_display(message, publish, currency, forum, capsys):
+ excluded.publish_display(None, None, message, publish, currency, forum)
+ captured = capsys.readouterr()
+ if not publish:
+ assert "message\n" == captured.out
+
+
+@pytest.mark.parametrize(
+ "currency, forum",
+ [("g1", "duniter"), ("g1", "monnaielibre"), ("g1-test", "duniter")],
+)
+def test_get_topic_id(currency, forum):
+ result = excluded.get_topic_id(currency, forum)
+ if currency == "g1":
+ if forum == "duniter":
+ topic_id = excluded.DUNITER_FORUM_G1_TOPIC_ID
+ else:
+ topic_id = excluded.MONNAIE_LIBRE_FORUM_G1_TOPIC_ID
+ else:
+ topic_id = excluded.DUNITER_FORUM_GTEST_TOPIC_ID
+
+
+@pytest.mark.parametrize(
+ "forum, response, topic_id",
+ [
+ ("duniter", {"topic_slug": "topic-slug"}, excluded.DUNITER_FORUM_G1_TOPIC_ID),
+ (
+ "monnaielibre",
+ {"topic_slug": "silkaj"},
+ excluded.MONNAIE_LIBRE_FORUM_G1_TOPIC_ID,
+ ),
+ (
+ "duniter",
+ {"topic_slug": "how-to-monnaie-libre"},
+ excluded.DUNITER_FORUM_G1_TOPIC_ID,
+ ),
+ ],
+)
+def test_publication_link(forum, response, topic_id, capsys):
+ if forum == "duniter":
+ forum_url = excluded.DUNITER_FORUM_URL
+ else:
+ forum_url = excluded.MONNAIE_LIBRE_FORUM_URL
+ expected = "Published on {}t/{}/{}/last\n".format(
+ forum_url, response["topic_slug"], str(topic_id)
+ )
+ excluded.publication_link(forum, response, topic_id)
+ captured = capsys.readouterr()
+ assert expected == captured.out
--
GitLab