.cargo/config

@@ -5,4 +5,7 @@ cn = "check --manifest-path neon/native/Cargo.toml"
 dex = "run --release --package duniter-dbex --"
 ta = "test --all"
 rr = "run --release --"
+uc = "update -p duniter-core"
+ug = "update -p duniter-gva"
+ugc = "update -p duniter-gva-conf"
 xtask = "run --package xtask --"

.dockerignore

+# Do not edit this file. It is generated from this command:
+# ./dockerignore.make
+
+.cargo
+.git*
+doc
+dockerignore.make
+gui
+test
+
+# ------------------
+# .gitignore content
+# ------------------
+
+*.sublime*
+node_modules/
+*.html
+npm-debug.log
+bin/jpgp*.jar
+.idea/
+gui/nw
+
+# Vim swap files
+*~
+*.swp
+*.swo
+
+# Vagrant
+.vagrant/
+vagrant/*.log
+vagrant/duniter
+
+# Python compiled
+*.pyc
+
+# Releases
+/work
+*.deb
+*.tar.gz
+*.log
+*.exe
+
+# vscode
+.vscode
+
+# istanbul
+.nyc_output
+coverage/
+
+# typecode
+typedoc/
+
+# files generated by tsc
+/index.js*
+/index.d.ts
+/server.js*
+/server.d.ts
+*/**/*.js*
+app/**/*.d.ts
+neon/lib/*.d.ts
+test/**/*.d.ts
+
+# files generated by neon
+neon/native/artifacts.json
+
+# rust binaries
+bin/duniter
+neon/native/index.node
+target
+
+# files generated by rust tests
+neon/native/tests/*.txt
+neon/native/tests/wotb-*
+test2.bin.gz
+**/*.wot

.gitlab-ci.yml

@@ -13,17 +13,19 @@ workflow:
       - app/**/*
       - bin/duniter_js
       - neon/**/*
-      - releases/**/*
+      - release/**/*
       - rust-bins/**/*
       - rust-libs/**/*
       - test/**/*
       - .gitlab-ci.yml
+      - Cargo.toml
+      - Cargo.lock
       - index.ts
       - package.json
       - package-lock.json
       - server.ts
 
-.env: &env
+.env:
   image: registry.duniter.org/docker/duniter-ci:v0.2.0
   tags:
     - redshift
@@ -36,14 +38,6 @@ workflow:
     - nvm install 10
     - nvm use 10
 
-
-.cached_nvm: &cached_nvm
-  <<: *env
-  cache:
-    untracked: true
-    paths:
-      - node_modules/
-
 .rust_env:
   image: registry.duniter.org/docker/rust/rust-x64-stable-ci:latest
   tags:
@@ -52,33 +46,8 @@ workflow:
     - export PATH="$HOME/.cargo/bin:$PATH"
     - rustup show && rustc --version && cargo --version
 
-
-#pages:
-#  <<: *env
-#  stage: pages
-#  cache: {}
-#  script:
-#    - npm i
-#    - npm doc
-#    - mkdir -p public
-#    - cp .gitlab/pages/pages-index.html public/index.html
-#    - sed -i "s/{BRANCH}/$CI_COMMIT_REF_NAME/g" public/index.html
-#    - mv typedoc public/
-#    - echo "$CI_JOB_ID"
-#    - curl "https://git.duniter.org/nodes/typescript/duniter/-/jobs/$CI_JOB_ID/artifacts/raw/coverage.tar.gz"
-#    - tar xzf coverage.tar.gz
-#    - mv coverage "public/coverage"
-#    - ls public
-#  artifacts:
-#    untracked: true
-#    paths:
-#      - public
-#  only:
-#    - loki
-#    - dev
-
 tests:
-  <<: *env
+  extends: .env
   rules:
     - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
       when: manual
@@ -124,9 +93,9 @@ audit_dependencies:
     - cargo deny -V
   stage: quality
   script:
-    - cargo deny check
+    - cargo deny --workspace check
 
-.integration_rules: &integration_rules
+.integration_rules:
   allow_failure: true
   rules:
     - if: $CI_COMMIT_TAG
@@ -134,24 +103,26 @@ audit_dependencies:
     - when: manual
 
 sync_g1:
-  <<: *env
-  <<: *integration_rules
+  extends:
+    - .env
+    - .integration_rules
   stage: integration
   script:
     - npm i
     - bash .gitlab/test/check_g1_sync.sh
 
 sync_gtest:
-  <<: *env
-  <<: *integration_rules
+  extends:
+    - .env
+    - .integration_rules
   stage: integration
   script:
     - npm i
     - bash .gitlab/test/check_gt_sync.sh
 
-.build_releases: &build_releases
+.build_releases:
   stage: package
-  image: duniter/release-builder:v1.4.0
+  image: duniter/release-builder:v2.1.0
   cache: {}
   tags:
     - redshift
@@ -160,7 +131,7 @@ sync_gtest:
       - work/bin/
 
 releases:test:
-  <<: *build_releases
+  extends: .build_releases
   allow_failure: true
   rules:
     - if: $CI_COMMIT_TAG
@@ -174,7 +145,7 @@ releases:test:
     expire_in: 72h
 
 releases:x64:
-  <<: *build_releases
+  extends: .build_releases
   rules:
     - if: $CI_COMMIT_TAG
   script:
@@ -186,18 +157,18 @@ releases:x64:
 
 .docker-build-app-image:
   stage: package
-  image: docker:18.06
+  image: docker:git
   tags:
-    - redshift
-  services:
-    - docker:18.06-dind
-  before_script:
-    - docker info
+    - docker
   script:
     - docker pull $CI_REGISTRY_IMAGE:$IMAGE_TAG || true
-    - docker build --cache-from $CI_REGISTRY_IMAGE:$IMAGE_TAG --pull -t "$CI_REGISTRY_IMAGE:$IMAGE_TAG" -f release/docker/Dockerfile .
-    - docker login -u "gitlab-ci-token" -p "$CI_BUILD_TOKEN" $CI_REGISTRY
-    - docker push "$CI_REGISTRY_IMAGE:$IMAGE_TAG"
+    - docker build --cache-from $CI_REGISTRY_IMAGE:$IMAGE_TAG --pull -t "$CI_REGISTRY_IMAGE:$IMAGE_TAG" --build-arg="INSTALL_DEX=$INSTALL_DEX" -f release/docker/Dockerfile .
+    #- docker login -u "gitlab-ci-token" -p "$CI_BUILD_TOKEN" $CI_REGISTRY
+    #- docker push "$CI_REGISTRY_IMAGE:$IMAGE_TAG"
+    # Temporary push on dockerhub 
+    - docker login -u "duniterteam" -p "$DUNITERTEAM_PASSWD"
+    - docker tag "$CI_REGISTRY_IMAGE:$IMAGE_TAG" "duniter/duniter:$IMAGE_TAG"
+    - docker push "duniter/duniter:$IMAGE_TAG"
 
 package:test:docker-test-image:
   extends: .docker-build-app-image
@@ -210,6 +181,7 @@ package:test:docker-test-image:
     - when: manual
   variables:
     IMAGE_TAG: "test-image"
+    INSTALL_DEX: "yes"
 
 package:dev:docker:
   extends: .docker-build-app-image
@@ -219,16 +191,15 @@ package:dev:docker:
     - if: $CI_COMMIT_BRANCH == "dev"
   variables:
     IMAGE_TAG: "dev"
+    INSTALL_DEX: "yes"
 
 package:prod:docker:
   stage: package
   rules:
     - if: $CI_COMMIT_TAG
-  image: docker:18.06
+  image: docker:git
   tags:
-    - redshift
-  services:
-    - docker:18.06-dind
+    - docker
   script:
     - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_TAG" -f release/docker/Dockerfile .
     - docker login -u "gitlab-ci-token" -p "$CI_BUILD_TOKEN" $CI_REGISTRY
@@ -240,7 +211,7 @@ package:prod:docker:
     - docker push duniter/duniter
 
 
-.release_jobs: &release_jobs
+.release_jobs:
   image: rdadev/jinja2:py3.6
   tags:
     - redshift
@@ -248,7 +219,7 @@ package:prod:docker:
     - python3 .gitlab/releaser
 
 prerelease:
-  <<: *release_jobs
+  extends: .release_jobs
   rules:
     - if: $CI_COMMIT_TAG
   stage: prerelease
@@ -257,7 +228,7 @@ prerelease:
     SOURCE_EXT: '["tar.gz", "zip"]'
 
 publish:
-  <<: *release_jobs
+  extends: .release_jobs
   rules:
     - if: $CI_COMMIT_TAG
       when: manual

.gitlab/releaser/releasenote.py

@@ -13,7 +13,8 @@ class ReleaseNote(ProjectApi):
     __PH_NOTE = PlaceHolder('note')
 
     def __init__(self):
-        ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
+        ProjectApi.__init__(self)
+        self.token = ('PRIVATE-TOKEN', os.environ['RELEASER_TOKEN'])
         self.message_read = False
 
     def get_note(self):
@@ -22,7 +23,7 @@ class ReleaseNote(ProjectApi):
         :return: The note if it exists, None otherwise.
         :rtype: str or None
         '''
-        request = self.build_request()
+        request = self.build_request('/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
         response = urllib.request.urlopen(request)
         response_data = response.read().decode()
         data = json.loads(response_data)
@@ -69,6 +70,10 @@ class ReleaseNote(ProjectApi):
             'description': note
         }
         send_data_serialized = json.dumps(send_data).encode('utf-8')
-        request = self.build_request('/release', data=send_data_serialized, method=method)
+        if not self.message_read:
+            request = self.build_request('/releases', data=send_data_serialized, method=method)
+        else:
+            request = self.build_request('/releases/{}'.format(os.environ['CI_COMMIT_TAG']), data=send_data_serialized, method=method)
         request.add_header('Content-Type', 'application/json')
+        request.add_header(*self.token)
         urllib.request.urlopen(request)

CONTRIBUTING.md

@@ -11,6 +11,10 @@ via [the technical forum] before making a change.
 
 Please note we have a specific workflow, please follow it in all your interactions with the project.
 
+## Developer documentation
+
+Please read [Developer documentation] before contribute.
+
 ## Workflow
 
 - You must create an issue for each feature you wish to develop, with a precise title and a
@@ -46,7 +50,8 @@ Please note we have a specific workflow, please follow it in all your interactio
 - @Moul
 - @c-geek
 
+[commit naming conventions]: ./doc/dev/git-conventions.md#naming-commits
+[Developer documentation]: ./doc/dev/index.md
+[project's git conventions]: ./doc/dev/git-conventions.md
 [Setting up your development environment]: ./doc/dev/setup_env_dev.md
 [the technical forum]: https://forum.duniter.org
-[project's git conventions]: ./doc/dev/git-conventions.md
-[commit naming conventions]: ./doc/dev/git-conventions.md#naming-commits

Cargo.toml

 [package]
 authors = ["elois <elois@duniter.org>"]
-description = "Duniter launcher."
+description = "Duniter cli."
 edition = "2018"
-keywords = ["duniter", "launcher"]
+keywords = ["duniter"]
 license = "AGPL-3.0"
-name = "duniter-launcher"
+name = "duniter-cli"
 repository = "https://git.duniter.org/nodes/typescript/duniter"
-version = "1.8.1"
+version = "1.9.0-dev"
 
 [[bin]]
 bench = false
-path = "rust-bins/duniter-launcher/src/main.rs"
+path = "rust-bins/duniter-cli/src/main.rs"
 name = "duniter"
 
 [dependencies]
@@ -18,6 +18,8 @@ anyhow = "1.0.32"
 ctrlc = "3.1.6"
 daemonize-me = "0.3.1"
 dirs = "3.0.1"
+duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core", features = ["bc-writer"] }
+duniter-gva-conf = { git = "https://git.duniter.org/nodes/rust/modules/duniter-gva" }
 log = "0.4.11"
 logwatcher = "0.1.1"
 nix = "0.17.0"
@@ -33,25 +35,18 @@ members = [
     "neon/native",
     "rust-bins/duniter-dbex",
     "rust-bins/xtask",
-    "rust-libs/dubp-wot",
-    "rust-libs/duniter-conf",
-    "rust-libs/duniter-dbs",
-    "rust-libs/duniter-bc-reader",
-    "rust-libs/duniter-dbs-write-ops",
-    "rust-libs/duniter-mempools",
-    "rust-libs/duniter-module",
     "rust-libs/duniter-server",
-    "rust-libs/modules/gva",
-    "rust-libs/modules/gva/dbs-reader",
-    "rust-libs/modules/gva/gql",
-    "rust-libs/modules/gva/indexer",
     "rust-libs/tests/duniter-integration-tests",
-    "rust-libs/tools/kv_typed"
 ]
 
+[patch.'https://git.duniter.org/nodes/rust/duniter-core']
+#duniter-core = { path = "../duniter-core" }
+
+[patch.'https://git.duniter.org/nodes/rust/modules/duniter-gva']
+#duniter-gva = { path = "../duniter-gva" }
+
 [patch.crates-io]
 #dubp = { git = "https://git.duniter.org/libs/dubp-rs-libs" }
-
 #dubp = { path = "../dubp-rs-libs" }
 
 #leveldb_minimal = { path = "../../../../rust/leveldb_minimal" }

README.md

@@ -18,22 +18,9 @@ However, we are running simultaneously a testing currency.
 
 See [Install a node documentation](https://duniter.org/en/wiki/duniter/install/).
 
-### Clients, wallets
+### Clients, wallets, GUI, Web-UI, Mobile-App
 
-#### Cesium
-
-- [Website](https://cesium.app/)
-- [Repository](https://git.duniter.org/clients/cesium-grp/cesium)
-
-#### Sakia
-
-- [Website](http://sakia-wallet.org)
-- [Repository](https://git.duniter.org/clients/python/sakia)
-
-#### Silkaj
-
-- [Website](https://silkaj.duniter.org)
-- [Repository](https://git.duniter.org/clients/python/silkaj)
+See [duniter.org](https://duniter.org/) or for end user [monnaie-libre.fr](https://monnaie-libre.fr/)
 
 ## Going further
 
@@ -42,6 +29,15 @@ See [Install a node documentation](https://duniter.org/en/wiki/duniter/install/)
 - See [CONTRIBUTING](./CONTRIBUTING.md).
 - [Guide (fr)](./doc/dev/contribute-french.md)
 
+#### In a post-it
+
+```bash
+git clone https://git.duniter.org/nodes/typescript/duniter.git
+cd duniter
+cargo xtask build
+./target/release/duniter start
+```
+
 ### Documentation
 
 Visit [Duniter website](https://duniter.org): it gathers theoretical informations, FAQ and several useful links. If you want to learn, this is the first place to visit.
@@ -65,15 +61,16 @@ The fact of migrating from code to [Rust] is commonly called "oxidation", so we
 
 The long-term goal is to oxidize Duniter entirely, but it is a long process that will take several years.
 
-Duniter is divided into several  git repositories:
+Duniter's code is divided into several git repositories:
 
-- [Duniter](https://git.duniter.org/nodes/typescript/duniter): this repository.
-- [Dubp-rs-libs](https://git.duniter.org/libs/dubp-rs-libs): Set of Rust libraries common to Duniter and a possible future Rust client/wallet.
-- [Web admin](https://git.duniter.org/nodes/typescript/modules/duniter-ui): web administration interface (optional).
-- [GVA](https://git.duniter.org/nodes/typescript/modules/gva-api): Future client API aimed to replace BMA. GVA stands for GraphQL Validation API.
+- **[dubp-rs-libs]** contains the logic common to Duniter and its clients.
+- **[duniter-core]** contains the core code of Duniter.
+- The gitlab subgroup **[nodes/rust/modules]** contains the main Duniter modules code (gva, admin, etc).
+- **[duniter]** repository contains the "official" implementations of the "duniter-cli" and "duniter-desktop" programs with their default modules (also contains the historical implementation being migrated).
 
-Optional repositories:
+Old optional repositories (will be archived when the migration is complete):
 
+- [**Web admin**](https://git.duniter.org/nodes/typescript/modules/duniter-ui): web administration interface (optional).
 - [Currency monit](https://git.duniter.org/nodes/typescript/modules/duniter-currency-monit): charts to monitor currency and web of trust state.
 - [Remuniter](https://github.com/duniter/remuniter): service to remunerate blocks issuers.
 

app/lib/common-libs/constants.ts

@@ -53,7 +53,7 @@ const CONDITIONS =
   "\\)|CSV\\(" +
   CSV_INTEGER +
   "\\))))*";
-
+const CONDITION_SIG_PUBKEY = "SIG\\((" + PUBKEY + ")\\)";
 const BMA_REGEXP = /^BASIC_MERKLED_API( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
 const BMAS_REGEXP = /^BMAS( ([a-z_][a-z0-9-_.]*))?( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))( (\/.+))?$/;
 const BMATOR_REGEXP = /^BMATOR( ([a-z0-9]{16})\.onion)( ([0-9.]+))?( ([0-9a-f:]+))?( ([0-9]+))$/;
@@ -535,6 +535,8 @@ export const CommonConstants = {
     LOCKTIME: find("Locktime: (" + INTEGER + ")"),
     INLINE_COMMENT: exact(COMMENT),
     OUTPUT_CONDITION: exact(CONDITIONS),
+    OUTPUT_CONDITION_SIG_PUBKEY: find(CONDITION_SIG_PUBKEY),
+    OUTPUT_CONDITION_SIG_PUBKEY_UNIQUE: exact(CONDITION_SIG_PUBKEY),
   },
   PEER: {
     BLOCK: find("Block: (" + INTEGER + "-" + FINGERPRINT + ")"),

app/lib/common-libs/parsers/block.ts

@@ -49,42 +49,42 @@ export class BlockParser extends GenericParser {
         { prop: "membersCount", regexp: CommonConstants.BLOCK.MEMBERS_COUNT },
         {
           prop: "identities",
-          regexp: /Identities:\n([\s\S]*)Joiners/,
+          regexp: /Identities:\n([\s\S]*?)Joiners/,
           parser: splitAndMatch("\n", CommonConstants.IDENTITY.INLINE),
         },
         {
           prop: "joiners",
-          regexp: /Joiners:\n([\s\S]*)Actives/,
+          regexp: /Joiners:\n([\s\S]*?)Actives/,
           parser: splitAndMatch("\n", CommonConstants.BLOCK.JOINER),
         },
         {
           prop: "actives",
-          regexp: /Actives:\n([\s\S]*)Leavers/,
+          regexp: /Actives:\n([\s\S]*?)Leavers/,
           parser: splitAndMatch("\n", CommonConstants.BLOCK.ACTIVE),
         },
         {
           prop: "leavers",
-          regexp: /Leavers:\n([\s\S]*)Excluded/,
+          regexp: /Leavers:\n([\s\S]*?)Excluded/,
           parser: splitAndMatch("\n", CommonConstants.BLOCK.LEAVER),
         },
         {
           prop: "revoked",
-          regexp: /Revoked:\n([\s\S]*)Excluded/,
+          regexp: /Revoked:\n([\s\S]*?)Excluded/,
           parser: splitAndMatch("\n", CommonConstants.BLOCK.REVOCATION),
         },
         {
           prop: "excluded",
-          regexp: /Excluded:\n([\s\S]*)Certifications/,
+          regexp: /Excluded:\n([\s\S]*?)Certifications/,
           parser: splitAndMatch("\n", CommonConstants.PUBLIC_KEY),
         },
         {
           prop: "certifications",
-          regexp: /Certifications:\n([\s\S]*)Transactions/,
+          regexp: /Certifications:\n([\s\S]*?)Transactions/,
           parser: splitAndMatch("\n", CommonConstants.CERT.OTHER.INLINE),
         },
         {
           prop: "transactions",
-          regexp: /Transactions:\n([\s\S]*)/,
+          regexp: /Transactions:\n([\s\S]*)/, // No need for greedy "?" regexp capture, "Transaction" parsing is different from previous multiline fields
           parser: extractTransactions,
         },
         { prop: "inner_hash", regexp: CommonConstants.BLOCK.INNER_HASH },

app/lib/common-libs/programOptions.ts

@@ -28,8 +28,6 @@ export interface ProgramOptions {
   loglevel?: string;
   sqlTraces?: boolean;
   memory?: boolean;
-  gva?: boolean;
-  noGva?: boolean;
 }
 
 export const cliprogram: ProgramOptions = {

app/lib/dal/indexDAL/abstract/IIndexDAO.ts

@@ -19,6 +19,8 @@ export interface IIndexDAO extends ReduceableDAO<IindexEntry> {
 
   searchThoseMatching(search: string): Promise<OldIindexEntry[]>;
 
+  getOldFromPubkey(pub: string): Promise<OldIindexEntry | null>;
+
   getFullFromUID(uid: string): Promise<FullIindexEntry>;
 
   getFullFromPubkey(pub: string): Promise<FullIindexEntry>;

app/lib/dal/indexDAL/leveldb/LevelDBBlockchain.ts

@@ -268,7 +268,7 @@ export class LevelDBBlockchain extends LevelDBTable<DBBlock>
     block.fork = false;
     // We remove the eventual fork
     const forkKey = LevelDBBlockchain.trimForkKey(block.number, block.hash);
-    if (this.forks.getOrNull(forkKey)) {
+    if (await this.forks.getOrNull(forkKey)) {
       await this.forks.del(forkKey);
     }
     // We return the saved block

app/lib/dal/indexDAL/leveldb/LevelDBIindex.ts

@@ -280,4 +280,12 @@ export class LevelDBIindex extends LevelDBTable<IindexEntry[]>
       .filter((u) => u.pub)
       .concat(pubIdentities.filter((p) => p.pub));
   }
+
+  async getOldFromPubkey(pub: string): Promise<OldIindexEntry | null> {
+    const identities = await this.findByPub(pub);
+    if (!identities.length) {
+      return null;
+    }
+    return OldTransformers.toOldIindexEntry(reduce(identities));
+  }
 }

app/lib/dal/sqliteDAL/IdentityDAL.ts

@@ -328,6 +328,10 @@ export class IdentityDAL extends AbstractSQLite<DBIdentity> {
     });
   }
 
+  findByPub(pub: string) {
+    return this.sqlFind({ pubkey: pub });
+  }
+
   async trimExpiredIdentities(medianTime: number) {
     await this.exec(
       "DELETE FROM " +