diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..a82cf4a944579329b64677917ddf7cc905c61396 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/.idea + +# plantuml JAR directory +/.local diff --git a/appendices/0003_RFC/model.puml b/appendices/0003_RFC/model.puml new file mode 100644 index 0000000000000000000000000000000000000000..6f6fa09021bb0305d57368f4b238dca63854319d --- /dev/null +++ b/appendices/0003_RFC/model.puml @@ -0,0 +1,60 @@ +@startuml + +class BlockNumberAndHash { + number: Int! + hash: String! +} + +class Block { + version: Int! + type: String! + currency: String! + issuer: String! + number: Int! + powMin: Int! + time: String! + mediantime: String! + membersCount: Int! + monetaryMass: Int! + unitBase: Int! + issuersCount: Int! + issuersFrame: Int! + issuersFrameVar: Int! + parameters: String + previousHash: String + dividend: Int + identities: [Identity!]! + joiners: [Membership!]! + actives: [Membership!]! + leavers: [Membership!]! + revoked: [CompactRevocation!]! + excluded: [String!]! + certifications: [CompactCertification!]! + transactions: [Transaction!]! + innerHash: String! + nonce: String! + signature: String! + hash: String! + raw: String! +} + +class Transaction { + version: Int! + type: String! + currency: String! + issuers: [String!]! + blockNumberAndHash: BlockNumberAndHash! + locktime: Int! + inputs: [CoinSource!]! + unlocks: [TxUnlock!]! + outputs: [TxOuput!]! + comment: String! + signatures: [String!]! + hash: String! + raw: String! + written_on: BlockNumberAndHash +} + +Block --> Transaction + +@enduml diff --git a/appendices/0003_RFC/model.svg b/appendices/0003_RFC/model.svg new file mode 100644 index 0000000000000000000000000000000000000000..2a0ed3b44f9fedf4347774ca6e138eb9d1ee8559 --- /dev/null +++ b/appendices/0003_RFC/model.svg @@ -0,0 +1,5 @@ +BlockNumberAndHashnumber: Int!hash: String!Blockversion: Int!type: String!currency: String!issuer: String!number: Int!powMin: Int!time: String!mediantime: String!membersCount: Int!monetaryMass: Int!unitBase: Int!issuersCount: Int!issuersFrame: Int!issuersFrameVar: Int!parameters: StringpreviousHash: Stringdividend: Intidentities: [Identity!]!joiners: [Membership!]!actives: [Membership!]!leavers: [Membership!]!revoked: [CompactRevocation!]!excluded: [String!]!certifications: [CompactCertification!]!transactions: [Transaction!]!innerHash: String!nonce: String!signature: String!hash: String!raw: String!Transactionversion: Int!type: String!currency: String!issuers: [String!]!blockNumberAndHash: BlockNumberAndHash!locktime: Int!inputs: [CoinSource!]!unlocks: [TxUnlock!]!outputs: [TxOuput!]!comment: String!signatures: [String!]!hash: String!raw: String!written_on: BlockNumberAndHash \ No newline at end of file diff --git a/appendices/0003_RFC/schema.graphql b/appendices/0003_RFC/schema.graphql new file mode 100644 index 0000000000000000000000000000000000000000..06d8dc410ff4f43275abb4504d8d537804961c3b --- /dev/null +++ b/appendices/0003_RFC/schema.graphql @@ -0,0 +1,408 @@ +# GraphQL schema for GVA API + +################################# +# Query +################################# + +type Query { + node: Node + current: Block + peers: Peers + heads: Heads + block(number: Int!): Block + blocks( + paging: Paging! + ): [Block!]! + identities( + paging: Paging! + issuer: String + usernameID: String + blockNumberAndHash: BlockNumberAndHashInput + state: DocumentState! + ): Identities + memberships( + paging: Paging! + issuer: String + state: DocumentState! + ): Memberships + certifications( + paging: Paging! + issuer: String + target: String + state: DocumentState! + ): Certifications + revocations( + paging: Paging! + issuer: String + state: DocumentState! + ): Revocations + transactions( + paging: Paging! + issuer: String + state: DocumentState! + ): Transactions + sources( + paging: Paging! + issuer: String + txOnly: Boolean! = false + udOnly: Boolean! = false + ): [CoinSource]! +} + +################################# +# Queries return types +################################# + +type Counters { + # Nombre total d'éléments + elements: Int! + # Total number of pages + pages: Int! +} + +type Peers { + counters: Counters! + peers: [Peer!]! +} + +type Heads { + counters: Counters! + peers: [Head!]! +} + +type Identities { + counters: Counters! + identities: [Identity!]! +} + +type Memberships { + counters: Counters! + identities: [Membership!]! +} + +type Revocations { + counters: Counters! + revocations: [Revocation!]! +} + +type Certifications { + counters: Counters! + certifications: [Certification!]! +} + +type Transactions { + counters: Counters! + transactions: [Transaction!]! +} + +################################# +# Inputs +################################# + +input Paging { + pageNumber: Int! = 0 + pageSize: Int! = 50 + fromBlock: Int! = 0 + # If toBlock is null, current block number is used + toBlock: Int +} + +input BlockNumberAndHashInput { + # Block number + number: Int! + # Block hash + hash: String! +} + +################################# +# Tools types +################################# + +type BlockNumberAndHash { + # Block number + number: Int! + # Block hash + hash: String! +} + +# Document state +enum DocumentState { + # Documetn written in blockchain + WRITTEN + # Document in sandbox (not written in blockhain) + PENDING + # Any documents + ANY +} + +################################# +# NODE types +################################# + +type Summary { + software: String! + version: String! + forkWindowSize: Int! +} + +type Sandbox { + # Depending on the implementation, the notion of "sandbox size" does not necessarily exist + size: Int + free: Int! +} + +type Sandboxes { + identities: Sandbox! + memberships: Sandbox! + certifications: Sandbox! + revocations: Sandbox! + transactions: Sandbox! +} + +type Node { + summary: Summary + peer: Peer, + head: Head, + sandboxes: Sandboxes +} + +################################# +# NETWORK types +################################# + +type Peer { + issuer: String!, + nodeId: Int!, + blockNumberAndHash: BlockNumberAndHash!, + endpoints: [String!]!, + signature: String!, +} + +type Head { + version: Int!, + currency: String!, + outgoingConf: Int!, + incomingConf: Int!, + freeMirrorRooms: Int!, + freeMemberRooms: Int!, + nodeId: Int!, + pubkey: String!, + blockNumberAndHash: BlockNumberAndHash!, + software: String!, + softVersion: String!, + signature: String!, + step: Int!, +} + +################################# +# WOT types +################################# + +enum MembershipType { + IN, + OUT +} + +type Membership { + version: Int! + type: String! + currency: String! + issuer: String! + blockNumberAndHash: BlockNumberAndHash! + membershipType: MembershipType! + idtyUsername: String! + idtyBlockNumberAndHash: BlockNumberAndHash! + signature: String! + raw: String! + # Null if the document is not written in blockchain + written_on: BlockNumberAndHash +} + +type Certification { + version: Int! + type: String! + currency: String! + issuer: String! + blockNumberAndHash: BlockNumberAndHash! + target: String! + idtyUsername: String! + idtyBlockNumberAndHash: BlockNumberAndHash! + idtySig: String! + signature: String! + raw: String! + # Null if the document is not written in blockchain + written_on: BlockNumberAndHash +} + +type Revocation { + version: Int! + type: String! + currency: String! + issuer: String! + idtyUsername: String! + idtyBlockNumberAndHash: BlockNumberAndHash! + idtySig: String! + signature: String! + raw: String! + # Null if the document is not written in blockchain + written_on: BlockNumberAndHash +} + +type Identity { + version: Int! + type: String! + currency: String! + issuer: String! + blockNumberAndHash: BlockNumberAndHash! + usernameID: String! + signature: String! + raw: String! + # Null if the document is not written in blockchain + written_on: BlockNumberAndHash + # True if the identity respects the distance rule, false if otherwise + # Some nodes may decide not to do this computation because it is time-consuming, so they return null + distance: Boolean! + # Membership + membership: Membership + # Pending memberships + pendMemberships: [Membership!]! + # Certifications received + recvCerts: [Certification!]! + # Certifications sent + sendCerts: [Certification!]! + # Identity revocation document (=null when identity not revoked) + revocation: Revocation +} + +################################# +# Transactions types (tx is alias for transaction) +################################# + +union CoinSource = UtxoInput | UdInput + +# UTXO Input (UTXO = Unused Transaction Ouput) +type UtxoInput { + amount: Int! + base: Int! + # transaction hash + hash: Int! + # output index + index: Int! +} + +# Universal Dividend input +type UdInput { + amount: Int! + base: Int! + pubkey: Int! + blockNumer: Int! +} + +# Transaction unlock source +type TxUnlock { + # Input index + index: Int! + # List of proof to unlock funds + unlocks: String! +} + +# Transaction output +type TxOuput { + amout: Int! + base: Int! + conditions: String! +} + +type Transaction { + version: Int! + type: String! + currency: String! + issuers: [String!]! + blockNumberAndHash: BlockNumberAndHash! + locktime: Int! + inputs: [CoinSource!]! + unlocks: [TxUnlock!]! + outputs: [TxOuput!]! + comment: String! + signatures: [String!]! + hash: String! + raw: String! + # Null if the document is not written in blockchain + written_on: BlockNumberAndHash +} + +################################# +# Block type +################################# + +# In the DUP protocol, the blocks contain a compacted version of the revocation document +type CompactRevocation { + issuer: String! + signature: String! +} + +# In the DUP protocol, the blocks contain a compacted version of the certification document +type CompactCertification { + issuer: String! + target: String! + blockNumber: Int! + signature: String! +} + +type Block { + version: Int! + type: String! + currency: String! + issuer: String! + number: Int! + powMin: Int! + time: String! + medianTime: Int! @deprecated(reason: "Use `blockchainTime`.") + blockchainTime: Int! + humanTime: Int! + membersCount: Int! + monetaryMass: Int! + unitBase: Int! + issuersCount: Int! + issuersFrame: Int! + issuersFrameVar: Int! + parameters: String + previousHash: String + dividend: Int + identities: [Identity!]! + joiners: [Membership!]! + actives: [Membership!]! + leavers: [Membership!]! + revoked: [CompactRevocation!]! + excluded: [String!]! + certifications: [CompactCertification!]! + transactions: [Transaction!]! + innerHash: String! + nonce: String! + signature: String! + hash: String! + raw: String! +} + +################################# +# Mutations +################################# + +# All mutations returns null if at least one document is invalid or if there is not enough space in the sandbox +type Mutation { + addIdentities ( + rawDocuments: [String!]! + ): [Identity!] + addMemberships ( + rawDocuments: [String!]! + ): [Membership!] + addCertifications ( + rawDocuments: [String!]! + ): [Certification!] + addRevocations ( + rawDocuments: [String!]! + ): [Revocation!] + addTransactions ( + rawDocuments: [String!]! + ): [Transaction!] +} diff --git a/plantuml.sh b/plantuml.sh new file mode 100755 index 0000000000000000000000000000000000000000..c8c8db797121d8af4b07d2242878fe68d539d39f --- /dev/null +++ b/plantuml.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +PROJECT_DIR=`pwd` +JAR_DIR="${PROJECT_DIR}/.local" +JAR_VERSION="1.2019.12" +JAR_FILE="${JAR_DIR}/plantuml-${JAR_VERSION}.jar" +JAR_URL="https://repo1.maven.org/maven2/net/sourceforge/plantuml/plantuml/${JAR_VERSION}/plantuml-${JAR_VERSION}.jar" + +# Make sure jar exists +checkJarExists() { + # Download the jar if need + if [[ ! -f "${JAR_FILE}" ]]; then + echo "Downloading PlantUML jar file: ${JAR_URL}..." + mkdir -p $JAR_DIR + wget -O $JAR_FILE ${JAR_URL} + if [[ $? -ne 0 ]]; then + echo "ERROR - Missing PlantUML JAR file at: $JAR_FILE" + echo " Please download it manually: ${JAR_URL}" + echo " and save it into the directory: ${JAR_DIR}" + exit 1 + fi + fi; +} + +generateSvg() { + echo Converting all PlatUML files from `pwd` into SVG... + java -jar ${JAR_FILE} -tsvg "./**.puml" -charset UTF-8 -progress -duration -nometadata + echo "Done" +} + +generatePng() { + echo Converting all PlatUML files from `pwd` into PNG... + java -jar ${JAR_FILE} -tpng "./**.puml" -charset UTF-8 -progress -duration -nometadata + echo "Done" +} + +case "$1" in +svg) + checkJarExists + generateSvg + + exit 0 +;; + +png) + checkJarExists + generatePng + + exit 0 +;; + +version) + java -jar ${JAR_FILE} -version + exit 0 +;; + + +help) + echo "Usage: $0 {svg|png|version|help}" + exit 0 +;; + +*) + $0 help + exit 0 +esac diff --git a/rfc/0003 RFC GraphQL API for Duniter Clients.md b/rfc/0003 RFC GraphQL API for Duniter Clients.md new file mode 100644 index 0000000000000000000000000000000000000000..a7b72dc80909d62d2f45cdcfa01f29c02f89c63c --- /dev/null +++ b/rfc/0003 RFC GraphQL API for Duniter Clients.md @@ -0,0 +1,380 @@ +# GraphQL API for Duniter Clients + +```txt +RFC: 3 +Title: GraphQL API for Duniter Clients +Type: New API module dedicated for client's needs +Parent: +Status: WIP +Author: vtexier +Created: 2018-01-19 +Last edited: 2018-01-19 +License: AGPL-3 +``` + +> This document is currently being written and thus it's not complete. Specifications may change in the future. + +## Table of Contents + +* [Abstract](#abstract) +* [GraphQL Verification API (aka GVA)](#graphql-verification-api-aka-gva) +* [Duniter Ecosystem](#duniter-ecosystem) +* [API standard](#api-standard) + * [Specifications](#specifications) + * [GraphQL License](#graphql-license) +* [Server Librairies](#server-librairies) + * [Javascript library](#javascript-library) + * [Rust Library](#rust-library) + * [Other languages](#other-languages) +* [Client Libraries](#client-libraries) +* [Requests and answers proposal](#requests-and-answers-proposal) + * [Requests process](#requests-process) + * [General form of the requests](#general-form-of-the-requests) + * [State requests](#state-requests) +* [Types](#types) + * [Identity example](#identity-example) + * [Identities data](#identities-data) +* [Duniter Server module](#duniter-server-module) + * [ENDPOINTS format](#endpoints-format) +* [API GraphQL Schema](#api-graphql-schema) + +## Abstract + +In the current Duniter server, the old BMA api is deprecated and will be removed. +The WS2P API is dedicated for the inter-node communication. + +We propose here a new API Module dedicated to client software. +It should deliver efficiently all information needed without slowing down the Duniter node. + +## GraphQL Verification API (aka GVA) + +This is the name proposed for this API. + +Why ? + +- We use **GraphQL**. +- We make data changes **Verification**. +- This is an **API** ;-) + +## Duniter Ecosystem + +As you know, there are two types of clients in Duniter ecosystem : + +**Naive clients:** + +These clients trust a given node and only one node. It trusts the administrator of the node. In case of cheating or in case of a bug, the client can be mistaken. The advantage of this behaviour is that the client is really smooth and fast. It doesn’t consume too much bandwith, and is compatible with small ressources endpoints. + +**Simplified Verification clients:** + +These clients trust the data they get because they request many nodes. They requests nodes identified by an identity, and as such, they are not sybillizable. The advantage of this behaviour is safety : the client request any data against multiple nodes. If one is cheating or buggy, its data wont match the data of other nodes. In such a case, this data will be ignored + + +The problem with current API (Basic Merkled API) is that it’s not really merklized : when you require data, you can only get every things, to its smallest details. The consequence is that when you need to verify data, you can only request it again and again against multiple nodes, resulting in endless loading. + +## API standard + +It is important that we use an already known standard to avoid difficulty in contribution. And to get all the evolution and ideas coming from the community supporting the standard. + +The proposed standard is [GraphQL](http://graphql.org/). Created by Facebook, it is used by many companies around the world. + +The PRO: https://blog.risingstack.com/graphql-overview-getting-started-with-graphql-and-nodejs/ +The CON: https://blog.hitchhq.com/graphql-3-reasons-not-to-use-it-7715f60cb934 + +### Specifications + +http://facebook.github.io/graphql + +### GraphQL License + +The specification of the API is licensed under the Open Web Fundation: + +The various implementations, for example the javascript one, are under the MIT license. + +https://code.facebook.com/posts/121714468491809/relicensing-the-graphql-specification/ + +## Server Librairies + +GraphQl use queries that need to be parsed by the server. So the use of a library avoid to write ourselves the base code. + +### Javascript library + +Facebook Javascript server library is available [here](https://github.com/graphql/graphql-js) + +### Rust Library + +Rust server library is available [here](https://github.com/graphql-rust/juniper) + +### Other languages + +http://graphql.org/code/#server-libraries + +## Client Libraries + +Python (Sakia, Silkaj) and Javascript (Cesium) have client libraries: + +http://graphql.org/code/#graphql-clients + +## Requests and answers proposal + +### Requests process + +1. The client sends a “Data Request” to a given node. It gets its answer, formed as a tree of data. +2. The client builds the Merkled Tree corresponding to this tree +3. The client send a “Verification request” to N given nodes. The verification request returns the root of the merkle tree. +4. The client compares the Root received by the verification nodes to the one he built, and validate or unvalidate the data + +These requests can be done in parallel. They should be sent to nodes that are on the same blockchain HEAD. + +Verification only concerns written state of data. Validity of the data can be insured thanks to hashes and signatures. + +### General form of the requests + +- The requests should be done using GraphQL (via HTTP or Websockets) +- The URI endpoint is unique. +- The Query should handle : + + The type of data requested + + The source of data : from the pools or from the blockchain. Data from the pools cannot be verified using a merkle tree : there are not guarantees that they are the same on every nodes. + + Filters (range, uid, pubkey, blockstamp, txhash…) + + The type of request (Data request or Verification) + +### State requests + +These requests have a merkle tree associated. They should be used to verify the state of a given data. + +They can be used on data in the blockchain or in the pool, as below : + +- If the data is found in the **blockchain**, its written field has a value and the hash is computed depending on this value +- If the data is found in the **pool**, its written field is empty. The hash is the hash of an empty string : e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +- If the data is **not found**, it returns a 404 (fixme: replace with GraphQL empty response) + +## Types + +We can request different types: + +- **Data:** a GraphQL json representation of the type of state requested from a document or an aggregation of documents. +- **Raw:** a GraphQL json containing a raw document. +- **Merkle Tree:** a GraphQL json containing the root of the merkle tree from a state. +- **Lists:** a GraphQL json containing a list of state or others types enumerated above. Lists are limited to a maximum number by default and must be paginated. + +### Identity example + +#### Identities data + +These data are requested for a given triplet of `(uid/pubkey/blockstamp)`. + +These data are not the raw data of an Identity. Only the convenient data for a client software. + +The data we need when we request Identity informations are : + + Is it a member ? + Its signature + Its blockchain state : written or not-written + Its distance state : outdistanced or not + Its sentry state : sentry or not + Its membership state : if a document was sent, which one (IN or OUT) + If IN document + at which block + at which timestamp + Its revokation state : revoked or not revoked + If revoked : + at which block + at which timestamp + +**GraphQL Request:** + +Parameters are mandatory, this is the key to retrieve the Identity. +Fields are optional like in any GrahQL query. + +```graphql +query getIdentity { + Identity(uid: "john", pubkey: "z7rDt7...", blockstamp: "22765-1AD84...") { + member + signature + written + outdistanced + sentry + Membership + Revoke + } +} +``` + +**GraphQL Response:** + +```json +{ + "data": { + "member": True, + "signature": "c4zYpECnNa99xc8Np/6NXXSo6bK7eGVAdG8QcZE8ZhFZeZjCv0F1Zzeub1ZDnWm4/TvKK9PjpuQBv9tBn9hcAA==", + "written": True + "outdistanced": False, + "sentry": True, + "Membership": { + "type": "IN", + "blockstamp": "22965-7FD42...", + "timestamp": "229658678", + }, + "Revoke": { + "revoked": True, + "blockstamp": "22965-7FD42...", + "timestamp": "229658678", + } + } +} +``` + +To get the full raw document type: + +```graphql +query getIdentityRaw { + IdentityRaw(uid: "john", pubkey: "z7rDt7...", blockstamp: "22765-1AD84...") +} +``` + +Returns: + +```json +{ + "data": { + "IdentityRaw": "Version: 10\n + Type: Identity\n + Currency: CURRENCY_NAME\n + Issuer: PUBLIC_KEY\n + UniqueID: USER_ID\n + Timestamp: BLOCK_UID\n + SIGNATURE" + } +} +``` + +**Calculate the Merkle Root Hash:** + +To be able to change version or add fields easily on data fields more often than raw document, +merkle check is done on raw document fields. + +```javascript +{ + result: Hash( + Hash(Hash(member), + + Hash(Hash(revocation.blockstamp), + Hash(revocation.timestamp), + Hash(revocation.signature) + Hash(Hash(revocation.written.blockstamp), + Hash(revocation.written.timestamp))? + )?, + Hash(expired), + Hash(outdistanced), + Hash(sentry)), + Hash(Hash(issuance.blockstamp),Hash(issuance.timestamp)), + Hash(Hash(written.blockstamp),Hash(written.timestamp)) + ) +} +``` + +Send the verification request: + +```graphql +query getIdentityMerkleTree { + IdentityMerkleTree(uid: "john", pubkey: "z7rDt7...", blockstamp: "22765-1AD84...") +} +``` + +Return the root of the merkle tree: + +```json +{ + "data": { + "IdentityMerkleTree": "478D46A98F75..." + } +} +``` + +**Query a list of identities UID:** + +For all list queries, an offset and a limit variable are specified by default. +You can use these to paginate your list and avoid a "timebomb" request. + +*A "timebomb" request is a request on an infinitly growing list of entities, +leading to slower and bigger responses that can, at the end, crash the server.* + +```graphql +# Query with variables and default values +query getIdentities($offset: Int = 0, $limit: Int = 1000) { + Identities { + uid + } +} +``` + +Variables values for the request (page 2 of the list, with 1000 elements per page) + +```json +{ + "offset": 1000, + "limit": 1000 +} +``` + +List response: + +```json +{ + "data": { + "Identities": [ + { + "uid": "Alice" + }, + { + "uid": "Bob" + } + ] +} +``` + +**Send an Identity Document:** + +GraphQL use another type of query to modifiy server's data. +It is called "mutation". + +As arguments, you must use "input objects" defined in the schema on the server. + +```graphql +# Send a write mutation +mutation sendIdentityDocument($identityRaw: InputIdentityRaw!) { + Identity +} +``` + +Variables: + +```json +{ + "identityRaw": "Version: 10\n + Type: Identity\n + Currency: CURRENCY_NAME\n + Issuer: PUBLIC_KEY\n + UniqueID: USER_ID\n + Timestamp: BLOCK_UID\n + SIGNATURE" +} +``` + +## Duniter Server module + +The GraphQL API is added on the server as a Duniter module. + +### ENDPOINTS format + +We follow the WS2P RFC proposal: https://git.duniter.org/nodes/common/doc/blob/ws2p_v2/rfc/0006_ws2p_v2.md#endpoints + +The endpoint format needs one path for the HTTP requests (queries and mutations) and one path for the websocket channel (subscriptions). + +So the websoket channel path is automatically deducted from the optionnal path: `path/subscriptions` + +## API GraphQL Schema + +See [schema.gql](../appendices/0003_RFC/schema.graphql) + +--- +End of the document.