diff --git a/.env b/.env
deleted file mode 100644
index 2f89c7773d17ab18d89978b440348d363fe8b2e0..0000000000000000000000000000000000000000
--- a/.env
+++ /dev/null
@@ -1,2 +0,0 @@
-prod=false
-LEVELDB_PATH = "./leveldb"
diff --git a/Dockerfile b/Dockerfile
index dde92426084185a0e963f6ec0f7960e00aec65c0..d7a1f851cd146fa51ecfb00b1bf836558cbe7028 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,30 +1,9 @@
-FROM python:3.9.18-bullseye as build_g1_output
-
-WORKDIR /app
+# this Dockerfile creates an image used by Duniter CI to run py-g1-migrator
+FROM python:3.9.18-bullseye
+# install libleveldb required by plyvel
RUN apt-get update && \
- DEBIAN_FRONTEND=noninteractive apt-get install -y sqlite3 libleveldb-dev jq
-
-WORKDIR /dump
-RUN curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz
-RUN tar xvzf g1-dump.tgz
-RUN rm g1-dump.tgz
-RUN mv tmp/backup-g1-duniter-1.8.7 duniter_default
-
-WORKDIR /py-g1-migrator
-COPY . .
-RUN rm -rf inputs/*
+ DEBIAN_FRONTEND=noninteractive apt-get install -y sqlite3 libleveldb-dev
+# install python requirements
RUN pip install -r requirements.txt
-
-ENV LEVELDB_PATH="/dump/duniter_default/data/leveldb"
-RUN ./main.py > output/main.log
-RUN sqlite3 /dump/duniter_default/txs.db --json "select time,comment,issuers,outputs from txs;" > inputs/transactions_history.json 2>> inputs/txs.err
-RUN ./generate_transactions_history.py
-RUN jq -s "{ identities: .[0].identities, wallets: .[0].wallets, initial_monetary_mass: .[0].initial_monetary_mass, current_block: .[0].current_block, transactions_history: .[1] }" output/gtest_genesis.json output/history.json > output/g1-data.json
-RUN mkdir /out
-
-# Run example from sources root
-# docker build -t py-g1-migrator-image -f Dockerfile .
-# docker run --rm -it -v "$(pwd)/output":/out py-g1-migrator-image cp output/g1-data.json /out
-# cp output/g1-data.json ~/dev/duniter-v2s/resources/g1-data.json
\ No newline at end of file
diff --git a/README.md b/README.md
index 5d052e7ad0f165913083ebff5514ac2f6a65df43..ffc2e2fff3be62cfb9c3c6183dca7e7ae466b982 100644
--- a/README.md
+++ b/README.md
@@ -1,62 +1,59 @@
# Data migration from Ğ1 v1 to Ğ1 v2
-## Configuration
-
-Edit `.env` file to suit your needs, then export environment variables:
-
- export $(xargs <.env)
-
## Execution
-This is used in Duniter CI:
+This program is used in Duniter CI:
https://git.duniter.org/nodes/rust/duniter-v2s/-/blob/56998122e42afd2c2c1642a72a6772a82490ccda/.gitlab-ci.yml#L270-L298
----
-
-Tested with python 3.8 and 3.9 and 3.10 and 3.11
-
-this tool allows you to transform current Ğ1 v1 data into genesis Ğ v2.
-
-It also allows importing up-to-date Ğ1 v1 history for your indexer v2s.
-
-You will need `substrate-interface` pip package:
-
- pip install -r requirements.txt
-or
-
- pip install substrate-interface
-
-## 1. Generate your up-to-date v2s Ğ1data genesis
-
- ./main.py
-
-This is making all the parsing and formating to generate final ĞTest genesis JSON.
-
-Inputs data are downloaded from https://g1-migrator.axiom-team.fr, which is provided by `dex` every days at 6:30 am.
-
-This take about 5 seconds, depends on your internet connection and CPU.
-
-After that:
-
-
- $ ls output/
- gdev.json
-
-
-You can optionally add the provisional launch timestamp of your blockchain, to adapt the expiration dates of the identities:
-
- ./main.py 1665373345
-
-## 2. Start you new Duniter v2s node with up-to-date Ğ1v1 datas !
-
- rm -rf output/chains && docker compose up
-
-Congratulations ! Your ĞTest blockchain starts.<br>
-You can explore it with: https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer
-
-## 3. (optional) Generate your up-to-date v2s-indexer Ğ1 transactions history
-
- ./generate_transactions_history.py
-
-Then you can start a new v2s-indexer with your new `output/history.json` json file.
+With the docker image produced by Dockerfile:
+
+```sh
+docker buildx build . -t h30x/py-g1-migrator
+docker image push h30x/py-g1-migrator
+```
+
+## Dev
+
+On your Duniter node
+
+```sh
+# create temporary directory
+mkdir /tmp/backup-g1-duniter-1.8.7
+# copy database ~ 1.5 GB
+cp -R $HOME/.config/duniter/duniter_default/data /tmp/backup-g1-duniter-1.8.7/
+# compress it
+tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
+# make it available with http (here it's available with https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz)
+mv /tmp/backup-g1-duniter-1.8.7.tgz /var/www/files.coinduf.eu
+```
+
+In your `py-g1-migrator` folder
+
+
+```sh
+# fetch database dump, extract, and move to your input folder
+curl https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz -o inputs/g1-dump.tgz
+tar xvzf inputs/g1-dump.tgz -C inputs
+mv inputs/tmp/backup-g1-duniter-1.8.7 inputs/duniter_default
+
+# use a python virtual environment, install requirements, and set env var to tell where the database is
+python -m venv env
+source ./env/bin/activate
+pip install -r requirements.txt
+export LEVELDB_PATH="./inputs/duniter_default/data/leveldb"
+# --- MAIN ---
+# main script outputs ./output/genesis.json which is used to build Duniter genesis state
+./main.py
+
+# --- SQUID ---
+# squid scripts are used by Duniter-Squid to provide seamless history for client users
+./squid-block.py # ./output/block_hist.json
+./squid-cert.py # ./output/cert_hist.json
+./squid-tx.py # ./output/tx_hist.json
+
+# make artifacts available to other (this will be done by the CI)
+scp ./output/block_hist.json wolf:/var/www/files.coinduf.eu/
+scp ./output/tx_hist.json wolf:/var/www/files.coinduf.eu/
+scp ./output/cert_hist.json wolf:/var/www/files.coinduf.eu/
+```
diff --git a/generate_transactions_history.py b/generate_transactions_history.py
deleted file mode 100755
index c7b0f8c6861fac8d67d8febb45b6b8d6379b9f50..0000000000000000000000000000000000000000
--- a/generate_transactions_history.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python3.9
-
-# Copyright (C) 2022 Axiom-Team.
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-from lib.utility import *
-
-print("Generate v2s-indexer up-to-date Ğ1 history transactions")
-
-history_final = {}
-history_brut = load_json_url('inputs/transactions_history.json')
-
-for transaction in history_brut:
- written_time = transaction['time']
- issuer_v1 = transaction['issuers'].replace('"','').replace('\\','').replace('[','').replace(']','').split(',')[0]
- output = transaction['outputs'].replace('"','').replace('\\','').replace('[','').replace(']','').split(':')
- amount = output[0]
- receiver_v1 = output[2].split('SIG(')[1].split(')')[0]
-
- # print(issuer_v1)
-
- issuer_pubkey_bytes = base58.b58decode(issuer_v1)
- issuer_pubkey_lenght = len(issuer_pubkey_bytes)
- receiver_pubkey_bytes = base58.b58decode(receiver_v1)
- receiver_pubkey_lenght = len(receiver_pubkey_bytes)
-
- issuer = issuer_v1
- receiver = receiver_v1
-
- comment = transaction['comment']
-
- if receiver not in history_final: history_final.update({receiver: []})
- history_final[receiver].append({'issuer': issuer, 'amount': amount, 'written_time': written_time, 'comment': comment})
-
-
-# print(history_final)
-
-
-history_final_json = json.dumps(history_final, indent=2).encode()
-history_json = open('output/history.json', 'wb')
-history_json.write(history_final_json)
diff --git a/lib/functions.py b/lib/functions.py
index c3bb4b84d586b3dcfb4d6098f512e6ba3040fc8e..c7124db412ea0429d095d15c5043a61ddb60bb63 100644
--- a/lib/functions.py
+++ b/lib/functions.py
@@ -1,4 +1,3 @@
-import base58
import json
import math
import collections
@@ -9,11 +8,9 @@ from adapters.duniter_v18.identities import LevelDBIdentitiesRepository
from adapters.duniter_v18.memberships import LevelDBMembershipsRepository
from adapters.duniter_v18.wallets import LevelDBWalletsRepository
from adapters.duniter_v18.ud_value import LevelDBUDValueRepository
-from lib.utility import load_json
-from lib.utility_param import *
# Constant to estimate cert interval
-CERT_PERIOD = b_days(5) # 5 days
+CERT_PERIOD = 432000 # 5 * 24 * 3600 # 5 days
# when iterating on blocks, log current block every NOTIF_INTERVAL
NOTIF_INTERVAL = 100000
@@ -93,8 +90,6 @@ def get_identities_and_wallets(start_timestamp: int, leveldb_path: str) -> tuple
print(f"missing money (added to treasury): {missing_money:,}")
# add missing money to treasury
treasury += missing_money
- # FIXME get real treasury address
- # wallets["5EYCAe5ijiYfyeZ2JJCGq56LmPyNRAKzpG4QkoQkkQNB5e6Z"] = treasury
# TODO make sure that index respects order of arrival
# Get identities names by pubkey
@@ -148,7 +143,7 @@ def get_identities_and_wallets(start_timestamp: int, leveldb_path: str) -> tuple
# timestamp of cert creation
created_at = blocks_repository.get(cert["created_on"])["medianTime"]
# block of next issuable cert
- next_issuable_on = created_at + CERT_PERIOD # TODO check if Duniter expects timestamp or block number
+ next_issuable_on = created_at + CERT_PERIOD
# timestamp of cert expiration
cert_expire_at = cert["expires_on"]
cert_expire_on = cert_expire_at
diff --git a/lib/utility.py b/lib/utility.py
deleted file mode 100644
index abd1bd8477ef23c3f6289dba29f20847a46dfe33..0000000000000000000000000000000000000000
--- a/lib/utility.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import json, base58
-from urllib.request import urlopen
-from substrateinterface import Keypair, KeypairType
-
-
-def v1_pubkey_to_v2_address(pubkey):
- pubkey_bytes = base58.b58decode(pubkey)
- pubkey_length = len(pubkey_bytes)
-
- # Add 0 head byte to pubkey so as to reach 32 bytes
- if pubkey_length < 32:
- pubkey_bytes = bytes([0] * (32 - pubkey_length)) + pubkey_bytes
-
- # get incomplete Substrate keypair (only public key) from public key bytes
- # FIXME use correct prefix (not 42)
- keypair = Keypair(
- public_key=pubkey_bytes, ss58_format=42, crypto_type=KeypairType.ED25519
- )
-
- # return V2 address
- return keypair.ss58_address
-
-
-def load_json(data):
- get_data = open(data)
- return json.load(get_data)
-
-
-def load_json_url(url):
- with open(".env", "r") as file:
- env = file.read()
-
- if "prod=true" in env:
- path = "/home/axiom/dex-data/"
- f = open(path + url)
- else:
- website = "https://g1-migrator.axiom-team.fr/"
- f = urlopen(website + url)
- return json.load(f)
diff --git a/lib/utility_param.py b/lib/utility_param.py
deleted file mode 100644
index 62f4ac9c4405f85b6ebeae276d24b2ab8383f973..0000000000000000000000000000000000000000
--- a/lib/utility_param.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Utility params
-
-def b_seconds(seconds: int) -> int:
- """converts a number of seconds to a number of 6-seconds blocs
- use lower approximation
- example : 2 seconds will be block 0
- example : 7 seconds will be block 1
- """
- return int(seconds / 6)
-
-def b_minutes(minutes: int) -> int:
- return b_seconds(minutes * 60)
-
-def b_hours(hours: int) -> int:
- return b_minutes(hours) * 60
-
-def b_days(days: int) -> int:
- return b_hours(days) * 24
\ No newline at end of file
diff --git a/note.md b/note.md
deleted file mode 100644
index ae76887705e5e9cccb9647f3db0fd2912ecef98a..0000000000000000000000000000000000000000
--- a/note.md
+++ /dev/null
@@ -1,42 +0,0 @@
-
-Note Hugo pour dev en local.
-
-## Sur mon noeud Duniter
-
-```sh
-mkdir /tmp/backup-g1-duniter-1.8.7
-cp -R $HOME/.config/duniter/duniter_default/data /tmp/backup-g1-duniter-1.8.7/
-# cp -R $HOME/.config/duniter/duniter_default/g1 /tmp/backup-g1-duniter-1.8.7/
-cp -R $HOME/.config/duniter/duniter_default/txs.db /tmp/backup-g1-duniter-1.8.7/
-tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
-mv /tmp/backup-g1-duniter-1.8.7.tgz /var/www/files.coinduf.eu
-```
-
-## Sur ma machine de dev dans py-g1-migrator
-
-```sh
-# récupérer le dump
-curl https://files.coinduf.eu/backup-g1-duniter-1.8.7.tgz -o inputs/g1-dump.tgz
-tar xvzf inputs/g1-dump.tgz -C inputs
-mv inputs/tmp/backup-g1-duniter-1.8.7 inputs/duniter_default
-
-# exec
-python -m venv env
-source ./env/bin/activate
-pip install -r requirements.txt
-export LEVELDB_PATH="./inputs/duniter_default/data/leveldb"
-# --- MAIN ---
-# main script outputs ./output/genesis.json which is used to build Duniter genesis state
-./main.py
-
-# --- SQUID ---
-# squid scripts are used by Duniter-Squid to provide seamless history for client users
-./squid-block.py # ./output/block_hist.json
-./squid-cert.py # ./output/cert_hist.json
-./squid-tx.py # ./output/tx_hist.json
-
-# copy to artifacts
-scp ./output/block_hist.json wolf:/var/www/files.coinduf.eu/
-scp ./output/tx_hist.json wolf:/var/www/files.coinduf.eu/
-scp ./output/cert_hist.json wolf:/var/www/files.coinduf.eu/
-```
diff --git a/requirements.txt b/requirements.txt
index ca3dce0c59a8c411f7fe613c31edf60db6aa64c3..91af2074cbdb154710c5179f739a34680c7caee6 100755
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1 @@
-substrate-interface
-base58
plyvel
diff --git a/scripts/convert_pubkey.py b/scripts/convert_pubkey.py
deleted file mode 100644
index 637b0a3ba4040319a6790f3feea6305e2a680343..0000000000000000000000000000000000000000
--- a/scripts/convert_pubkey.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env python3
-
-from lib.utility import v1_pubkey_to_v2_address
-import sys
-
-if len(sys.argv) < 2:
- print("Please give your v1 pubkey as argument")
- sys.exit(1)
-
-pubkey: str = sys.argv[1]
-
-print(v1_pubkey_to_v2_address(pubkey))