[feat] #256: Introduce excluded command for DeathReaper

Introduce some tests: TODO write all tests

silkaj/cli.py

@@ -28,6 +28,7 @@ from silkaj.constants import (
     G1_TEST_DEFAULT_ENDPOINT,
     SILKAJ_VERSION,
 )
+from silkaj.excluded import excluded
 from silkaj.license import license_command
 from silkaj.membership import send_membership
 from silkaj.money import cmd_amount
@@ -124,6 +125,7 @@ cli.add_command(list_blocks)
 cli.add_command(send_certification)
 cli.add_command(checksum_command)
 cli.add_command(difficulties)
+cli.add_command(excluded)
 cli.add_command(transaction_history)
 cli.add_command(id_pubkey_correspondence)
 cli.add_command(currency_info)

silkaj/constants.py

@@ -18,12 +18,17 @@ G1_SYMBOL = "Ğ1"
 GTEST_SYMBOL = "ĞTest"
 G1_DEFAULT_ENDPOINT = "BMAS g1.duniter.org 443"
 G1_TEST_DEFAULT_ENDPOINT = "BMAS g1-test.duniter.org 443"
+
+ONE_HOUR = 3600
+
 SUCCESS_EXIT_STATUS = 0
 FAILURE_EXIT_STATUS = 1
+
 BMA_MAX_BLOCKS_CHUNK_SIZE = 5000
 PUBKEY_MIN_LENGTH = 43
 PUBKEY_MAX_LENGTH = 44
 PUBKEY_PATTERN = f"[1-9A-HJ-NP-Za-km-z]{{{PUBKEY_MIN_LENGTH},{PUBKEY_MAX_LENGTH}}}"
+
 MINIMAL_ABSOLUTE_TX_AMOUNT = 0.01
 MINIMAL_RELATIVE_TX_AMOUNT = 1e-6
 CENT_MULT_TO_UNIT = 100

silkaj/excluded.py

+# Copyright  2016-2021 Maël Azimi <m.a@moul.re>
+#
+# Silkaj is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Silkaj is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Silkaj. If not, see <https://www.gnu.org/licenses/>.
+
+import logging
+import sys
+import time
+import urllib
+
+import click
+import pendulum
+from duniterpy import constants as dp_const
+from duniterpy.api.bma import blockchain
+from duniterpy.api.client import Client
+from duniterpy.documents.block import Block
+from pydiscourse import DiscourseClient
+from pydiscourse.exceptions import DiscourseClientError
+
+from silkaj import constants, wot_tools
+from silkaj.blockchain_tools import BlockchainParams
+from silkaj.network_tools import ClientInstance
+from silkaj.tools import CurrencySymbol
+
+G1_CESIUM_URL = "https://demo.cesium.app/"
+GTEST_CESIUM_URL = "https://g1-test.cesium.app/"
+CESIUM_BLOCK_PATH = "#/app/block/"
+
+DUNITER_FORUM_URL = "https://forum.duniter.org/"
+MONNAIE_LIBRE_FORUM_URL = "https://forum.monnaie-libre.fr/"
+
+DUNITER_FORUM_G1_TOPIC_ID = 4393
+DUNITER_FORUM_GTEST_TOPIC_ID = 6554
+MONNAIE_LIBRE_FORUM_G1_TOPIC_ID = 17627
+
+
+@click.command(
+    "excluded",
+    help="Generate membership exclusions messages, \
+markdown formatted and publish them on Discourse Forums",
+)
+@click.option(
+    "-a",
+    "--api-id",
+    help="Username used on Discourse forum API",
+)
+@click.option(
+    "-du",
+    "--duniter-forum-api-key",
+    help="API key used on Duniter Forum",
+)
+@click.option(
+    "-ml",
+    "--ml-forum-api-key",
+    help="API key used for Monnaie Libre Forum",
+)
+@click.argument("days", default=1, type=click.FloatRange(0, 50))
+@click.option(
+    "--publish",
+    is_flag=True,
+    help="Publish the messages on the forums, otherwise print them",
+)
+def excluded(api_id, duniter_forum_api_key, ml_forum_api_key, days, publish):
+    params = BlockchainParams().params
+    currency = params["currency"]
+    check_options(api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency)
+    bma_client = ClientInstance().client
+    blocks_to_process = get_blocks_to_process(bma_client, days, params)
+    if not blocks_to_process:
+        no_exclusion(days, currency)
+    message = gen_message_over_blocks(bma_client, blocks_to_process, params)
+    if not message:
+        no_exclusion(days, currency)
+    header = gen_header(blocks_to_process)
+    # Add ability to publish just one of the two forum, via a flags?
+
+    publish_display(
+        api_id,
+        duniter_forum_api_key,
+        header + message,
+        publish,
+        currency,
+        "duniter",
+    )
+    if currency == dp_const.G1_CURRENCY_CODENAME:
+        publish_display(
+            api_id,
+            ml_forum_api_key,
+            header + message,
+            publish,
+            currency,
+            "monnaielibre",
+        )
+
+
+def check_options(api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency):
+    if publish and (
+        not api_id
+        or not duniter_forum_api_key
+        or (not ml_forum_api_key and currency != dp_const.G1_TEST_CURRENCY_CODENAME)
+    ):
+        sys.exit(
+            "Error: To be able to publish, api_id, duniter_forum_api, and \
+ml_forum_api_key (not required for {constants.GTEST_SYMBOL}) options should be specified"
+        )
+
+
+def no_exclusion(days, currency):
+    # Humanize
+    sys.exit(f"No exclusion to report within the last {days} day(s) on {currency}")
+
+
+def get_blocks_to_process(bma_client, days, params):
+    head_number = bma_client(blockchain.current)["number"]
+    block_number_days_ago = (
+        head_number - days * 24 * constants.ONE_HOUR / params["avgGenTime"]
+    )
+    # print(block_number_days_ago) # DEBUG
+
+    blocks_with_excluded = bma_client(blockchain.excluded)["result"]["blocks"]
+    for i, block_number in reversed(list(enumerate(blocks_with_excluded))):
+        if block_number < block_number_days_ago:
+            break
+    return blocks_with_excluded[i + 1 :]
+
+
+def gen_message_over_blocks(bma_client, blocks_to_process, params):
+    """
+    Loop over the list of blocks to retrieve and parse the blocks
+    Ignore revocation kind of exclusion
+    """
+    message = ""
+    for block_number in blocks_to_process:
+        logging.info(f"Processing block number {block_number}")
+        print(f"Processing block number {block_number}")
+        # DEBUG / to be removed once the #115 logging system is set
+
+        try:
+            block = bma_client(blockchain.block, block_number)
+        except urllib.error.HTTPError:
+            time.sleep(2)
+            block = bma_client(blockchain.block, block_number)
+        block_hash = block["hash"]
+        block = Block.from_signed_raw(block["raw"] + block["signature"] + "\n")
+
+        if block.revoked and block.excluded[0] == block.revoked[0].pubkey:
+            continue
+        message += await generate_message(block, block_hash, params)
+    return message
+
+
+def gen_header(blocks_to_process):
+    nbr_exclusions = len(blocks_to_process)
+    # Handle when there is one block with multiple exclusion within
+    # And when there is a revocation
+    s = "s" if nbr_exclusions > 1 else ""
+    des_du = "des" if nbr_exclusions > 1 else "du"
+    currency_symbol = CurrencySymbol().symbol
+    header = f"## Exclusion{s} de la toile de confiance {currency_symbol}, perte{s} {des_du} statut{s} de membre"
+    message_g1 = "\n> Message automatique. Merci de notifier vos proches de leur exclusion de la toile de confiance."
+    return header + message_g1
+
+
+def generate_message(block, block_hash, params):
+    """
+    Loop over exclusions within a block
+    Generate identity header + info
+    """
+    message = ""
+    for i, excluded in enumerate(block.excluded):
+        lookup = wot_tools.wot_lookup(excluded)[0]
+        uid = lookup["uids"][0]["uid"]
+
+        if params["currency"] == dp_const.G1_CURRENCY_CODENAME:
+            cesium_url = G1_CESIUM_URL
+        else:
+            cesium_url = GTEST_CESIUM_URL
+        cesium_url += CESIUM_BLOCK_PATH
+        message += f"\n\n### @{uid} [{uid}]({cesium_url}{block.number}/{block_hash}?ssl=true)\n"
+        message += generate_identity_info(lookup, block, params)
+    return message
+
+
+def generate_identity_info(lookup, block, params):
+    info = "- **Certifié·e par**"
+    nbr_different_certifiers = 0
+    for i, certifier in enumerate(lookup["uids"][0]["others"]):
+        if certifier["uids"][0] not in info:
+            nbr_different_certifiers += 1
+        info += elements_inbetween_list(i, lookup["uids"][0]["others"])
+        info += "@" + certifier["uids"][0]
+    if lookup["signed"]:
+        info += ".\n- **A certifié**"
+    for i, certified in enumerate(lookup["signed"]):
+        info += elements_inbetween_list(i, lookup["signed"])
+        info += "@" + certified["uid"]
+    dt = pendulum.from_timestamp(block.mediantime + constants.ONE_HOUR, tz="local")
+    info += ".\n- **Exclu·e le** " + dt.format("LLLL", locale="fr")
+    info += " CET\n- **Raison de l'exclusion** : "
+    if nbr_different_certifiers < params["sigQty"]:
+        info += "manque de certifications"
+    else:
+        info += "expiration du document d'adhésion"
+    # a renouveller tous les ans (variable) humanize(params[""])
+    return info
+
+
+def elements_inbetween_list(i, cert_list):
+    return " " if i == 0 else (" et " if i + 1 == len(cert_list) else ", ")
+
+
+def publish_display(api_id, forum_api_key, message, publish, currency, forum):
+    if publish:
+        topic_id = get_topic_id(currency, forum)
+        publish_message_on_the_forum(api_id, forum_api_key, message, topic_id, forum)
+    elif forum == "duniter":
+        click.echo(message)
+
+
+def get_topic_id(currency, forum):
+    if currency == dp_const.G1_CURRENCY_CODENAME:
+        if forum == "duniter":
+            return DUNITER_FORUM_G1_TOPIC_ID
+        else:
+            return MONNAIE_LIBRE_FORUM_G1_TOPIC_ID
+    else:
+        return DUNITER_FORUM_GTEST_TOPIC_ID
+
+
+def publish_message_on_the_forum(api_id, forum_api_key, message, topic_id, forum):
+    if forum == "duniter":
+        discourse_client = DiscourseClient(
+            DUNITER_FORUM_URL, api_username=api_id, api_key=forum_api_key
+        )
+    else:
+        discourse_client = DiscourseClient(
+            MONNAIE_LIBRE_FORUM_URL,
+            api_username=api_id,
+            api_key=forum_api_key,
+        )
+    try:
+        response = discourse_client.create_post(message, topic_id=topic_id)
+        publication_link(forum, response, topic_id)
+    except DiscourseClientError as error:
+        logging.error(f"Issue publishing on {forum} because: {error}")
+        # Handle DiscourseClient exceptions, pass them to the logger
+
+    # discourse_client.close()
+    # How to close this client? It looks like it is not implemented
+    # May be by closing requests' client
+
+
+def publication_link(forum, response, topic_id):
+    forum_url = DUNITER_FORUM_URL if forum == "duniter" else MONNAIE_LIBRE_FORUM_URL
+    print(f"Published on {forum_url}t/{response['topic_slug']}/{str(topic_id)}/last")

tests/test_excluded.py

+# Copyright  2016-2021 Maël Azimi <m.a@moul.re>
+#
+# Silkaj is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Silkaj is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with Silkaj. If not, see <https://www.gnu.org/licenses/>.
+
+import pytest
+from duniterpy.documents.block import Block
+
+from silkaj import excluded
+
+
+@pytest.mark.parametrize(
+    "api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency, error",
+    [
+        (None, None, None, False, "g1", False),
+        (None, None, None, True, "g1", True),
+        ("DeathReaper", None, None, True, "g1", True),
+        (None, "key", None, True, "g1", True),
+        (None, None, "key", True, "g1", True),
+        ("DeathReaper", "key", "key", True, "g1", False),
+        ("DeathReaper", "key", None, True, "g1-test", False),
+    ],
+)
+def test_check_options(
+    api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency, error, capsys
+):
+    if error:
+        with pytest.raises(SystemExit) as pytest_exit:
+            excluded.check_options(
+                api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency
+            )
+            assert pytest_exit.type == SystemExit
+            assert "Error: To be able to publish, api_id" in capsys.readouterr().out
+    else:
+        excluded.check_options(
+            api_id, duniter_forum_api_key, ml_forum_api_key, publish, currency
+        )
+        assert not capsys.readouterr().out
+
+
+"""
+def test_generate_identity_info():
+    ref = "- **Certifié·e par** @GildasMalassinet, @Marcolibri, @Gregory et @gpsqueeek.\n\
+- **A certifié** @lesailesbleues, @Marcolibri, @Gregory et @LaureBLodjtahne.\n\
+- **Exclu·e le** 2020-01-27 04:06:09\n\
+- **Raison de l'exclusion** : manque de certifications"
+    # missing lookup and block
+    # 292110
+    raw_block = "Version: 11\nType: Block\nCurrency: g1\nNumber: 292110\nPoWMin: 89\n\
+Time: 1580098477\nMedianTime: 1580094369\nUnitBase: 0\n\
+Issuer: D3krfq6J9AmfpKnS3gQVYoy7NzGCc61vokteTS8LJ4YH\nIssuersFrame: 161\n\
+IssuersFrameVar: 0\nDifferentIssuersCount: 32\n\
+PreviousHash: 0000009F88010580F54C0403BC87BF10E912D0FBC274294343DDC748F697991D\n\
+PreviousIssuer: 5P9HB1uBdE3ZqThSatBdBbjVmh3psa1BhsCRBumn1aMo\nMembersCount: 2527\n\
+Identities:\nJoiners:\nActives:\nLeavers:\nRevoked:\nExcluded:\n\
+DKWXiEEd4wkf4pgd95vkGnNkHxzTKDpPPSgMay5WJ7Fc\nCertifications:\nTransactions:\n\
+InnerHash: CD17661EBD858CC374918D2F94363E22AB5BDC617FDEEA729C384616FD2285C4\n\
+Nonce: 10800000223569\n"
+    block_sig = "GvREqrzvbS2RHoLCEbfjjPcsJ+G+jVKqejF5fu9i3076XJ4pmc+udyCPoJ5TiCo7OPB5GOLmsCLocJZbqQS1CA=="
+    block = Block.from_signed_raw(raw_block + block_sig + "\n")
+    gen = excluded.generate_identity_info(lookup, block, 5)
+    assert gen == ref
+"""
+
+
+def test_elements_inbetween_list():
+    list = ["toto", "titi", "tata"]
+    assert excluded.elements_inbetween_list(0, list) == " "
+    assert excluded.elements_inbetween_list(1, list) == ", "
+    assert excluded.elements_inbetween_list(2, list) == " et "
+
+
+@pytest.mark.parametrize(
+    "message, publish, currency, forum", [("message", False, "g1", "duniter")]
+)
+def test_publish_display(message, publish, currency, forum, capsys):
+    excluded.publish_display(None, None, message, publish, currency, forum)
+    captured = capsys.readouterr()
+    if not publish:
+        assert "message\n" == captured.out
+
+
+@pytest.mark.parametrize(
+    "currency, forum",
+    [("g1", "duniter"), ("g1", "monnaielibre"), ("g1-test", "duniter")],
+)
+def test_get_topic_id(currency, forum):
+    result = excluded.get_topic_id(currency, forum)
+    if currency == "g1":
+        if forum == "duniter":
+            topic_id = excluded.DUNITER_FORUM_G1_TOPIC_ID
+        else:
+            topic_id = excluded.MONNAIE_LIBRE_FORUM_G1_TOPIC_ID
+    else:
+        topic_id = excluded.DUNITER_FORUM_GTEST_TOPIC_ID
+
+
+@pytest.mark.parametrize(
+    "forum, response, topic_id",
+    [
+        ("duniter", {"topic_slug": "topic-slug"}, excluded.DUNITER_FORUM_G1_TOPIC_ID),
+        (
+            "monnaielibre",
+            {"topic_slug": "silkaj"},
+            excluded.MONNAIE_LIBRE_FORUM_G1_TOPIC_ID,
+        ),
+        (
+            "duniter",
+            {"topic_slug": "how-to-monnaie-libre"},
+            excluded.DUNITER_FORUM_G1_TOPIC_ID,
+        ),
+    ],
+)
+def test_publication_link(forum, response, topic_id, capsys):
+    if forum == "duniter":
+        forum_url = excluded.DUNITER_FORUM_URL
+    else:
+        forum_url = excluded.MONNAIE_LIBRE_FORUM_URL
+    expected = "Published on {}t/{}/{}/last\n".format(
+        forum_url, response["topic_slug"], str(topic_id)
+    )
+    excluded.publication_link(forum, response, topic_id)
+    captured = capsys.readouterr()
+    assert expected == captured.out