docker image for CI and more cleanup

.env

-prod=false
-LEVELDB_PATH = "./leveldb"

Dockerfile

-FROM python:3.9.18-bullseye as build_g1_output
+# this Dockerfile creates an image used by Duniter CI to run py-g1-migrator
+FROM python:3.9.18-bullseye
 
 WORKDIR /app
+COPY ./requirements.txt .
 
+# install libleveldb required by plyvel
 RUN apt-get update && \
-    DEBIAN_FRONTEND=noninteractive apt-get install -y sqlite3 libleveldb-dev jq
-
-WORKDIR /dump
-RUN curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz
-RUN tar xvzf g1-dump.tgz
-RUN rm g1-dump.tgz
-RUN mv tmp/backup-g1-duniter-1.8.7 duniter_default
-
-WORKDIR /py-g1-migrator
-COPY . .
-RUN rm -rf inputs/*
+    DEBIAN_FRONTEND=noninteractive apt-get install -y libleveldb-dev
 
+# install python requirements
 RUN pip install -r requirements.txt
-
-ENV LEVELDB_PATH="/dump/duniter_default/data/leveldb"
-RUN ./main.py > output/main.log
-RUN sqlite3 /dump/duniter_default/txs.db --json "select time,comment,issuers,outputs from txs;" > inputs/transactions_history.json 2>> inputs/txs.err
-RUN ./generate_transactions_history.py
-RUN jq -s "{ identities: .[0].identities, wallets: .[0].wallets, initial_monetary_mass: .[0].initial_monetary_mass, current_block: .[0].current_block, transactions_history: .[1] }" output/gtest_genesis.json output/history.json > output/g1-data.json
-RUN mkdir /out
-
-# Run example from sources root
-# docker build -t py-g1-migrator-image -f Dockerfile .
-# docker run --rm -it -v "$(pwd)/output":/out py-g1-migrator-image cp output/g1-data.json /out
-# cp output/g1-data.json ~/dev/duniter-v2s/resources/g1-data.json
\ No newline at end of file

README.md

 # Data migration from Ğ1 v1 to Ğ1 v2
 
-## Configuration
-
-Edit `.env` file to suit your needs, then export environment variables:
-
-    export $(xargs <.env)
-
 ## Execution
 
-This is used in Duniter CI:
+This program is used in Duniter CI:
 
 https://git.duniter.org/nodes/rust/duniter-v2s/-/blob/56998122e42afd2c2c1642a72a6772a82490ccda/.gitlab-ci.yml#L270-L298
 
----
+With the docker image produced by Dockerfile:
 
-Tested with python 3.8 and 3.9 and 3.10 and 3.11
+```sh
+docker buildx build . -t h30x/py-g1-migrator
+docker image push h30x/py-g1-migrator
+```
 
-this tool allows you to transform current Ğ1 v1 data into genesis Ğ v2.
+## Dev
 
-It also allows importing up-to-date Ğ1 v1 history for your indexer v2s.
+On your Duniter node
 
-You will need `substrate-interface` pip package:
+```sh
+# create temporary directory
+mkdir /tmp/backup-g1-duniter-1.8.7
+# copy database ~ 1.5 GB
+cp -R $HOME/.config/duniter/duniter_default/data /tmp/backup-g1-duniter-1.8.7/
+# compress it
+tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
+# make it available with http (here it's available with https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz)
+mv /tmp/backup-g1-duniter-1.8.7.tgz /var/www/files.coinduf.eu
+```
 
-    pip install -r requirements.txt
-or
+In your `py-g1-migrator` folder
 
-    pip install substrate-interface
 
-## 1. Generate your up-to-date v2s Ğ1data genesis
+```sh
+# fetch database dump, extract, and move to your input folder
+curl https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz -o inputs/g1-dump.tgz
+tar xvzf inputs/g1-dump.tgz  -C inputs
+mv inputs/tmp/backup-g1-duniter-1.8.7 inputs/duniter_default
 
+# use a python virtual environment, install requirements, and set env var to tell where the database is
+python -m venv env
+source ./env/bin/activate
+pip install -r requirements.txt
+export LEVELDB_PATH="./inputs/duniter_default/data/leveldb" 
+# --- MAIN ---
+# main script outputs ./output/genesis.json which is used to build Duniter genesis state
 ./main.py
 
-This is making all the parsing and formating to generate final ĞTest genesis JSON.
-
-Inputs data are downloaded from https://g1-migrator.axiom-team.fr, which is provided by `dex` every days at 6:30 am.
-
-This take about 5 seconds, depends on your internet connection and CPU.
-
-After that:
-
-
-    $ ls output/
-    gdev.json
-
-
-You can optionally add the provisional launch timestamp of your blockchain, to adapt the expiration dates of the identities:
-
-    ./main.py 1665373345
-
-## 2. Start you new Duniter v2s node with up-to-date Ğ1v1 datas !
-
-    rm -rf output/chains && docker compose up
-
-Congratulations ! Your ĞTest blockchain starts.<br>
-You can explore it with: https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer
-
-## 3. (optional) Generate your up-to-date v2s-indexer Ğ1 transactions history
-
-    ./generate_transactions_history.py
-
-Then you can start a new v2s-indexer with your new `output/history.json` json file.
+# --- SQUID ---
+# squid scripts are used by Duniter-Squid to provide seamless history for client users
+./squid-block.py # ./output/block_hist.json
+./squid-cert.py # ./output/cert_hist.json
+./squid-tx.py # ./output/tx_hist.json
+
+# make artifacts available to other (this will be done by the CI)
+scp ./output/block_hist.json wolf:/var/www/files.coinduf.eu/
+scp ./output/tx_hist.json wolf:/var/www/files.coinduf.eu/
+scp ./output/cert_hist.json wolf:/var/www/files.coinduf.eu/
+```

generate_transactions_history.py

-#!/usr/bin/env python3.9
-
-#  Copyright (C) 2022 Axiom-Team.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-
-from lib.utility import *
-
-print("Generate v2s-indexer up-to-date Ğ1 history transactions")
-
-history_final = {}
-history_brut = load_json_url('inputs/transactions_history.json')
-
-for transaction in history_brut:
-    written_time = transaction['time']
-    issuer_v1 = transaction['issuers'].replace('"','').replace('\\','').replace('[','').replace(']','').split(',')[0]
-    output = transaction['outputs'].replace('"','').replace('\\','').replace('[','').replace(']','').split(':')
-    amount = output[0]
-    receiver_v1 = output[2].split('SIG(')[1].split(')')[0]
-    
-    # print(issuer_v1)
-    
-    issuer_pubkey_bytes = base58.b58decode(issuer_v1)
-    issuer_pubkey_lenght = len(issuer_pubkey_bytes)
-    receiver_pubkey_bytes = base58.b58decode(receiver_v1)
-    receiver_pubkey_lenght = len(receiver_pubkey_bytes)
-    
-    issuer = issuer_v1
-    receiver = receiver_v1
-    
-    comment = transaction['comment']
-    
-    if receiver not in history_final: history_final.update({receiver: []})
-    history_final[receiver].append({'issuer': issuer, 'amount': amount, 'written_time': written_time, 'comment': comment})
-    
-
-# print(history_final)
-
-
-history_final_json = json.dumps(history_final, indent=2).encode()
-history_json = open('output/history.json', 'wb')
-history_json.write(history_final_json)

lib/functions.py

-import base58
 import json
 import math
 import collections
@@ -9,11 +8,9 @@ from adapters.duniter_v18.identities import LevelDBIdentitiesRepository
 from adapters.duniter_v18.memberships import LevelDBMembershipsRepository
 from adapters.duniter_v18.wallets import LevelDBWalletsRepository
 from adapters.duniter_v18.ud_value import LevelDBUDValueRepository
-from lib.utility import load_json
-from lib.utility_param import *
 
 # Constant to estimate cert interval
-CERT_PERIOD = b_days(5)  # 5 days
+CERT_PERIOD = 432000  # 5 * 24 * 3600  # 5 days
 
 # when iterating on blocks, log current block every NOTIF_INTERVAL
 NOTIF_INTERVAL = 100000
@@ -93,10 +90,7 @@ def get_identities_and_wallets(start_timestamp: int, leveldb_path: str) -> tuple
         print(f"missing money (added to treasury): {missing_money:,}")
         # add missing money to treasury
         treasury += missing_money
-    # FIXME get real treasury address
-    # wallets["5EYCAe5ijiYfyeZ2JJCGq56LmPyNRAKzpG4QkoQkkQNB5e6Z"] = treasury
 
-    # TODO make sure that index respects order of arrival
     # Get identities names by pubkey
     for pubkey, identity in identities_repository:
         index = identity["wotb_id"] + 1
@@ -148,7 +142,7 @@ def get_identities_and_wallets(start_timestamp: int, leveldb_path: str) -> tuple
             # timestamp of cert creation
             created_at = blocks_repository.get(cert["created_on"])["medianTime"]
             # block of next issuable cert
-            next_issuable_on = created_at + CERT_PERIOD # TODO check if Duniter expects timestamp or block number
+            next_issuable_on = created_at + CERT_PERIOD
             # timestamp of cert expiration
             cert_expire_at = cert["expires_on"]
             cert_expire_on = cert_expire_at
@@ -177,11 +171,12 @@ def get_blocks(leveldb_path: str) -> list:
             True
             and not block.get("certifications")
             and not block.get("transactions")
-            and not block.get("joiners") # TODO membership events
-            and not block.get("leavers") # TODO membership events
-            and not block.get("revoked") # TODO membership events
-            and not block.get("actives") # TODO membership events
-            and not block.get("excluded") # TODO membership events
+            # TODO handle membership events below
+            and not block.get("joiners")
+            and not block.get("leavers")
+            and not block.get("revoked")
+            and not block.get("actives")
+            and not block.get("excluded")
         )
         sample = {
             "height": block.get("number"),
@@ -253,8 +248,10 @@ def get_cert(leveldb_path: str) -> list:
     # initialize
     CERTVALIDITY = 63115200  # 3600 * 24 * 365.25 * 2 validity of certification in seconds (2 years)
     cert_should_expire = {}  # maps (issuer, receiver) to expiration timestamp
-    may_expire = collections.deque() # queue of (expire, (issuer, receiver)), should be ordered by expire timestamp
-    cert_events = [] # cert events returned by this function (Creation, Renewal, Removal)
+    # queue of (expire, (issuer, receiver)), should be ordered by expire timestamp
+    may_expire = collections.deque()
+    # cert events returned by this function (Creation, Renewal, Removal)
+    cert_events = []
     identity_id = {}  # maps pubkey to identity index
     blockMedianTime = []  # medianTime of the block n at position n
     # repos

lib/utility.py

-import json, base58
-from urllib.request import urlopen
-from substrateinterface import Keypair, KeypairType
-
-
-def v1_pubkey_to_v2_address(pubkey):
-    pubkey_bytes = base58.b58decode(pubkey)
-    pubkey_length = len(pubkey_bytes)
-
-    # Add 0 head byte to pubkey so as to reach 32 bytes
-    if pubkey_length < 32:
-        pubkey_bytes = bytes([0] * (32 - pubkey_length)) + pubkey_bytes
-
-    # get incomplete Substrate keypair (only public key) from public key bytes
-    # FIXME use correct prefix (not 42)
-    keypair = Keypair(
-        public_key=pubkey_bytes, ss58_format=42, crypto_type=KeypairType.ED25519
-    )
-
-    # return V2 address
-    return keypair.ss58_address
-
-
-def load_json(data):
-    get_data = open(data)
-    return json.load(get_data)
-
-
-def load_json_url(url):
-    with open(".env", "r") as file:
-        env = file.read()
-
-    if "prod=true" in env:
-        path = "/home/axiom/dex-data/"
-        f = open(path + url)
-    else:
-        website = "https://g1-migrator.axiom-team.fr/"
-        f = urlopen(website + url)
-    return json.load(f)

lib/utility_param.py

-# Utility params
-
-def b_seconds(seconds: int) -> int:
-    """converts a number of seconds to a number of 6-seconds blocs
-    use lower approximation
-    example : 2 seconds will be block 0
-    example : 7 seconds will be block 1
-    """
-    return int(seconds / 6)
-
-def b_minutes(minutes: int) -> int:
-    return b_seconds(minutes * 60)
-
-def b_hours(hours: int) -> int:
-    return b_minutes(hours) * 60
-
-def b_days(days: int) -> int:
-    return b_hours(days) * 24
\ No newline at end of file

note.md

-
-Note Hugo pour dev en local.
-
-## Sur mon noeud Duniter
-
-```sh
-mkdir /tmp/backup-g1-duniter-1.8.7
-cp -R $HOME/.config/duniter/duniter_default/data /tmp/backup-g1-duniter-1.8.7/
-# cp -R $HOME/.config/duniter/duniter_default/g1 /tmp/backup-g1-duniter-1.8.7/
-cp -R $HOME/.config/duniter/duniter_default/txs.db /tmp/backup-g1-duniter-1.8.7/
-tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
-mv /tmp/backup-g1-duniter-1.8.7.tgz /var/www/files.coinduf.eu
-```
-
-## Sur ma machine de dev dans py-g1-migrator
-
-```sh
-# récupérer le dump
-curl https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz -o inputs/g1-dump.tgz
-tar xvzf inputs/g1-dump.tgz  -C inputs
-mv inputs/tmp/backup-g1-duniter-1.8.7 inputs/duniter_default
-
-# exec
-python -m venv env
-source ./env/bin/activate
-pip install -r requirements.txt
-export LEVELDB_PATH="./inputs/duniter_default/data/leveldb" 
-# --- MAIN ---
-# main script outputs ./output/genesis.json which is used to build Duniter genesis state
-./main.py
-
-# --- SQUID ---
-# squid scripts are used by Duniter-Squid to provide seamless history for client users
-./squid-block.py # ./output/block_hist.json
-./squid-cert.py # ./output/cert_hist.json
-./squid-tx.py # ./output/tx_hist.json
-
-# copy to artifacts
-scp ./output/block_hist.json wolf:/var/www/files.coinduf.eu/
-scp ./output/tx_hist.json wolf:/var/www/files.coinduf.eu/
-scp ./output/cert_hist.json wolf:/var/www/files.coinduf.eu/
-```

requirements.txt

-substrate-interface
-base58
 plyvel

scripts/convert_pubkey.py

-#!/usr/bin/env python3
-
-from lib.utility import v1_pubkey_to_v2_address
-import sys
-
-if len(sys.argv) < 2:
-    print("Please give your v1 pubkey as argument")
-    sys.exit(1)
-
-pubkey: str = sys.argv[1]
-
-print(v1_pubkey_to_v2_address(pubkey))