Mise à jour de GitLab prévue ce samedi 23 octobre 2021 à partir de 9h00 CET

Commit 86ab7796 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

Merge branch 'loki' into dev

parents e85a4432 bb0ee690
app/cli.js
app/lib/blockchain/*.js
app/lib/blockchain/interfaces/*.js
app/lib/computation/*.js
app/lib/common-libs/*.js
app/lib/common-libs/**/*.js
app/lib/db/*.js
app/lib/dto/*.js
app/lib/indexer.js
app/lib/common.js
app/lib/dal/drivers/*.js
app/lib/dal/sqliteDAL/*.js
app/lib/dal/sqliteDAL/index/*.js
app/lib/dal/fileDALs/*.js
app/lib/dal/fileDAL.js
app/service/*.js
app/lib/rules/*.js
app/lib/system/*.js
app/lib/streams/*.js
app/lib/helpers/*.js
app/modules/ws2p/*.js
app/modules/ws2p/lib/*.js
app/modules/ws2p/lib/*/*.js
app/lib/*.js
app/modules/wizard.js
app/modules/router.js
app/modules/revert.js
app/modules/reset.js
app/modules/reapply.js
app/modules/peersignal.js
app/modules/plugin.js
app/modules/daemon.js
app/modules/export-bc.js
app/modules/check-config.js
app/modules/config.js
app/modules/prover/*.js
app/modules/prover/lib/*.js
app/modules/keypair/*.js
app/modules/keypair/lib/*.js
app/modules/bma/*.js
app/modules/bma/lib/*.js
app/modules/bma/lib/entity/*.js
app/modules/bma/lib/controllers/*.js
app/modules/crawler/*.js
app/modules/crawler/lib/*.js
app/ProcessCpuProfiler.js
app/lib/common/package.js
app/*.js
app/**/*.js
test/*.js
test/**/*.js
\ No newline at end of file
{
"parserOptions": {
"ecmaVersion": 8
},
"plugins": [
"mocha"
],
// "ecmaFeatures": {
// "arrowFunctions": true,
// "generators": true
// },
"ecmaFeatures": {
"arrowFunctions": true,
"generators": true
},
"rules": {
"curly": 0,
......
......@@ -35,48 +35,14 @@ vagrant/duniter
.nyc_output
coverage/
# TS migration
test/blockchain/*.js*
test/blockchain/*.d.ts
test/blockchain/lib/*.js*
test/blockchain/lib/*.d.ts
# typecode
typedoc/
/index.js*
/index.d.ts
/server.js*
/server.d.ts
app/**/*.js*
app/**/*.d.ts
test/integration/revoked_pubkey_replay.js
test/integration/server-shutdown.js
test/integration/transactions-csv-cltv-sig.js
test/integration/ws2p*js
test/integration/*.js.map
test/integration/*.d.ts
test/integration/membership_chainability.js*
test/integration/membership_chainability.d.ts
test/integration/tools/toolbox.js*
test/integration/tools/toolbox.d.ts
test/integration/tools/TestUser.js*
test/integration/tools/TestUser.d.ts
test/integration/documents-currency.js*
test/integration/documents-currency.d.ts
test/integration/forwarding.js
test/integration/branches_switch.js
test/integration/branches2.js
test/integration/transactions-chaining.js
test/fast/modules/crawler/block_pulling.js*
test/fast/modules/crawler/block_pulling.d.ts
test/fast/fork*.js*
test/fast/fork*.d.ts
test/fast/proxies*.js*
test/fast/proxies*.d.ts
test/fast/modules/ws2p/*.js*
test/fast/modules/ws2p/*.d.ts
test/fast/modules/common/grammar.js*
test/fast/modules/common/grammar.d.ts
test/fast/prover/pow-1-cluster.d.ts
test/fast/prover/pow-1-cluster.js
test/fast/prover/pow-1-cluster.js.map
test/fast/protocol-local-rule-chained-tx-depth.js
test/fast/protocol-local-rule-chained-tx-depth.js.map
test/fast/protocol-local-rule-chained-tx-depth.d.ts
test/**/*.d.ts
test/**/*.js*
\ No newline at end of file
......@@ -2,6 +2,7 @@ stages:
- github-sync
- build
- test
- pages
- package
- prerelease
- release
......@@ -33,28 +34,81 @@ push_to_github:
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
.cached_nvm: &cached_nvm
<<: *nvm_env
cache:
untracked: true
paths:
- node_modules/
build:
<<: *nvm_env
<<: *cached_nvm
stage: build
script:
- yarn
pages:
<<: *nvm_env
stage: pages
cache: {}
script:
- yarn
- yarn doc
- mkdir -p public
- cp .gitlab/pages/pages-index.html public/index.html
- sed -i "s/{BRANCH}/$CI_COMMIT_REF_NAME/g" public/index.html
- mv typedoc public/
- echo "$CI_JOB_ID"
- curl "https://git.duniter.org/nodes/typescript/duniter/-/jobs/$CI_JOB_ID/artifacts/raw/coverage.tar.gz"
- tar xzf coverage.tar.gz
- mv coverage "public/coverage"
- ls public
artifacts:
untracked: true
paths:
- public
only:
- loki
- dev
test:
<<: *cached_nvm
stage: test
script:
- yarn test
# Push coverage to GitLab pages
- tar cvzf coverage.tar.gz coverage/
# Code coverage display in GitLab
- sed -n 23p coverage/index.html | grep -Po "\d+.\d+" | sed -e "s/\(.*\)/<coverage>\1%<\/coverage>/"
coverage: '/<coverage>(\d+.\d+\%)<\/coverage>/'
artifacts:
paths:
- coverage.tar.gz
expire_in: 4h
sync_g1:
<<: *nvm_env
stage: test
script:
- yarn
- yarn test
- sed -n 23p coverage/index.html
- bash .gitlab/test/check_g1_sync.sh
sync_gtest:
<<: *nvm_env
stage: test
script:
- yarn
- bash .gitlab/test/check_gt_sync.sh
.build_releases: &build_releases
stage: package
allow_failure: false
image: duniter/release-builder:v1.0.1
image: duniter/release-builder:v1.2.0
cache: {}
when: manual
tags:
- redshift-duniter-builder
when: manual
artifacts:
paths: &releases_artifacts
- work/bin/
......@@ -62,6 +116,7 @@ test:
releases:test:
<<: *build_releases
script:
- rm -rf node_modules/
- bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
artifacts:
paths: *releases_artifacts
......@@ -72,6 +127,7 @@ releases:test:
releases:x64:
<<: *build_releases
script:
- rm -rf node_modules/
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
artifacts:
paths: *releases_artifacts
......
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>CI pages</title>
<style>
html, body {
font-family: "Courier New", Consolas, monospace;
}
li {
line-height: 25px;
}
</style>
</head>
<body>
<p>
<img src="https://raw.github.com/duniter/duniter/master/images/250%C3%97250.png"/>
</p>
<h1>Continuous Integration (CI) pages:</h1>
<p>
<ul>
<li><a href="https://nodes.duniter.io/typescript/duniter/typedoc/">Typedoc (`{BRANCH}` branch)</a></li>
<li><a href="https://nodes.duniter.io/typescript/duniter/coverage/">Coverage (`{BRANCH}` branch)</a></li>
</ul>
</p>
</body>
</html>
\ No newline at end of file
#!/usr/bin/env bash
G1_TARGET_BLOCK=132446 # This is a fixed block# which determines to the sha1 hashes
G1_IINDEX_CS=26393b64cdb9abb8e4012d6914f475635cba4c60
G1_MINDEX_CS=7c5f07c7705647365b8965fcfc5a084c2f82a388
G1_CINDEX_CS=3803c1ed8d3dd8f31558666d8dfd30272a6d0b74
G1_SINDEX_CS=551bdba1855d5c49cd503fcb8ad787b2a24c2c42
.gitlab/test/check_indexes.sh /tmp/duniter_ci_dump/ g1 ${G1_TARGET_BLOCK} ${G1_IINDEX_CS} ${G1_MINDEX_CS} ${G1_CINDEX_CS} ${G1_SINDEX_CS}
#!/usr/bin/env bash
GT_TARGET_BLOCK=210000 # This is a fixed block# which determines to the sha1 hashes
GT_IINDEX_CS=dfd2dfc3d4d0ced4c101badb4d4a1ab85de8cbde
GT_MINDEX_CS=9d8f665f5fcf1f21082278c4787bb3df085ff109
GT_CINDEX_CS=b141361fb40f4c13f03f4640151c7674e190a4dd
GT_SINDEX_CS=7c6801027e39b9fea9be973d8773ac77d2c9a1f9
.gitlab/test/check_indexes.sh /tmp/duniter_ci_dump/ gt ${GT_TARGET_BLOCK} ${GT_IINDEX_CS} ${GT_MINDEX_CS} ${GT_CINDEX_CS} ${GT_SINDEX_CS}
#!/usr/bin/env bash
ORIGIN_DIR=`pwd`
mkdir -p $1
DUMP_DIR=`cd $1 && pwd`
CURRENCY=$2
ARCHIVES="$DUMP_DIR/archives_$CURRENCY"
DB_TEST="gitlab_ci_sync_test_$CURRENCY"
G1_TARGET_BLOCK=$3 # This is a fixed block# which determines to the sha1 hashes
G1_IINDEX_CS=$4
G1_MINDEX_CS=$5
G1_CINDEX_CS=$6
G1_SINDEX_CS=$7
checksum_test() {
local table=$1
local correct_hash=$2
local db=$3
echo "Checking $table's checksum..."
bin/duniter --mdb ${db} dump table "$table" > "$DUMP_DIR/$table"
result_hash=`sha1sum "$DUMP_DIR/$table" | grep -Po ".* " | grep -Po "[a-f0-9]+"`
# rm -f "$DUMP_DIR/$table"
if [ "$result_hash" == "$correct_hash" ]; then
echo "OK";
else
echo "Error! Wrong hash detected. ($result_hash != $correct_hash)"
exit 1
fi
}
sync_data() {
local db=$1
local target=$2
local target_block=$3
local reset_data="bin/duniter --mdb ${db} reset all"
local sync="bin/duniter --mdb ${db} sync ${target} --nointeractive ${target_block}"
echo "$reset_data"
${reset_data}
echo "$sync"
${sync}
}
if [ -d ${ARCHIVES} ]; then
echo "Updating archives..."
cd ${ARCHIVES}
git checkout master
git pull origin master
else
echo "Cloning archives..."
git clone https://git.duniter.org/c-geek/blockchain-archives.git ${ARCHIVES}
fi
echo "Positionnement dans $ORIGIN_DIR"
cd ${ORIGIN_DIR}
sync_data ${DB_TEST} "$ARCHIVES/$CURRENCY" ${G1_TARGET_BLOCK}
checksum_test i_index ${G1_IINDEX_CS} ${DB_TEST}
checksum_test m_index ${G1_MINDEX_CS} ${DB_TEST}
checksum_test c_index ${G1_CINDEX_CS} ${DB_TEST}
checksum_test s_index ${G1_SINDEX_CS} ${DB_TEST}
![Duniter logo](https://raw.github.com/duniter/duniter/master/images/250×250.png)
![Duniter logo](https://git.duniter.org/nodes/typescript/duniter/raw/dev/images/250%C3%97250.png)
# Duniter [![Build Status](https://api.travis-ci.org/duniter/duniter.png)](https://travis-ci.org/duniter/duniter) [![Coverage Status](https://coveralls.io/repos/github/duniter/duniter/badge.svg?branch=master)](https://coveralls.io/github/duniter/duniter?branch=master) [![Dependencies](https://david-dm.org/duniter/duniter.svg)](https://david-dm.org/duniter/duniter)
......@@ -6,7 +6,7 @@ Duniter (previously uCoin) is a libre software allowing to create a new kind of
Inspired by [Bitcoin](https://github.com/bitcoin/bitcoin) and [OpenUDC](https://github.com/Open-UDC/open-udc) projects.
<p align="center"><img src="https://github.com/duniter/duniter/blob/master/images/duniter_admin_g1.png" /></p>
<p align="center"><img src="https://git.duniter.org/nodes/typescript/duniter/raw/dev/images/duniter_admin_g1.png" /></p>
## Development state
......
......@@ -14,11 +14,20 @@
const SAMPLING_PERIOD = 150 // milliseconds
const MAX_SAMPLES_DISTANCE = 20 * 1000000 // seconds
function getMicrosecondsTime() {
export function getMicrosecondsTime() {
const [ seconds, nanoseconds ] = process.hrtime()
return seconds * 1000000 + nanoseconds / 1000
}
export function getNanosecondsTime() {
const [ seconds, nanoseconds ] = process.hrtime()
return seconds * 1000000 + nanoseconds
}
export function getDurationInMicroSeconds(before:number) {
return parseInt(String(getMicrosecondsTime() - before))
}
interface CpuUsage {
user: number
system:number
......
......@@ -74,6 +74,7 @@ export const ExecuteCommand = () => {
.option('--nohttplogs', 'Disable HTTP logs')
.option('--isolate', 'Avoid the node to send peering or status informations to the network')
.option('--forksize <size>', 'Maximum size of fork window', parseInt)
.option('--notrim', 'Disable the INDEX trimming.')
.option('--memory', 'Memory mode')
;
......
......@@ -11,9 +11,16 @@
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
import {MiscIndexedBlockchain} from "./MiscIndexedBlockchain"
import {IindexEntry, IndexEntry, Indexer, MindexEntry, SindexEntry} from "../indexer"
import {BlockchainOperator} from "./interfaces/BlockchainOperator"
import {
BasedAmount,
FullIindexEntry,
IindexEntry,
IndexEntry,
Indexer,
MindexEntry,
SimpleTxEntryForWallet,
SimpleUdEntryForWallet
} from "../indexer"
import {ConfDTO} from "../dto/ConfDTO"
import {BlockDTO} from "../dto/BlockDTO"
import {DBHead} from "../db/DBHead"
......@@ -25,16 +32,15 @@ import {CertificationDTO} from "../dto/CertificationDTO"
import {MembershipDTO} from "../dto/MembershipDTO"
import {TransactionDTO} from "../dto/TransactionDTO"
import {CommonConstants} from "../common-libs/constants"
import {FileDAL} from "../dal/fileDAL"
import {NewLogger} from "../logger"
import {DBTx} from "../db/DBTx"
import {Underscore} from "../common-libs/underscore"
import {OtherConstants} from "../other_constants"
const _ = require('underscore')
export class DuniterBlockchain {
export class DuniterBlockchain extends MiscIndexedBlockchain {
constructor(blockchainStorage:BlockchainOperator, dal:any) {
super(blockchainStorage, dal.mindexDAL, dal.iindexDAL, dal.sindexDAL, dal.cindexDAL)
}
static async checkBlock(block:BlockDTO, withPoWAndSignature:boolean, conf: ConfDTO, dal:any) {
static async checkBlock(block:BlockDTO, withPoWAndSignature:boolean, conf: ConfDTO, dal:FileDAL) {
const index = Indexer.localIndex(block, conf)
if (withPoWAndSignature) {
await CHECK.ASYNC.ALL_LOCAL(block, conf, index)
......@@ -55,7 +61,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G98
if (Indexer.ruleCurrency(block, HEAD) === false) throw Error('ruleCurrency');
// BR_G51
if (Indexer.ruleNumber(block, HEAD) === false) throw Error('ruleNumber');
if (Indexer.ruleNumber(block, HEAD) === false) {
throw Error('ruleNumber')
}
// BR_G52
if (Indexer.rulePreviousHash(block, HEAD) === false) throw Error('rulePreviousHash');
// BR_G53
......@@ -103,7 +111,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G70
if (Indexer.ruleCertificationToLeaver(cindex) === false) throw Error('ruleCertificationToLeaver');
// BR_G71
if (Indexer.ruleCertificationReplay(cindex) === false) throw Error('ruleCertificationReplay');
if (Indexer.ruleCertificationReplay(cindex) === false) {
throw Error('ruleCertificationReplay')
}
// BR_G72
if (Indexer.ruleCertificationSignature(cindex) === false) throw Error('ruleCertificationSignature');
// BR_G73
......@@ -123,7 +133,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G80
if (Indexer.ruleMembershipLeaverIsMember(mindex) === false) throw Error('ruleMembershipLeaverIsMember');
// BR_G81
if (Indexer.ruleMembershipActiveIsMember(mindex) === false) throw Error('ruleMembershipActiveIsMember');
if (Indexer.ruleMembershipActiveIsMember(mindex) === false) {
throw Error('ruleMembershipActiveIsMember')
}
// BR_G82
if (Indexer.ruleMembershipRevokedIsMember(mindex) === false) throw Error('ruleMembershipRevokedIsMember');
// BR_G83
......@@ -133,7 +145,9 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// BR_G85
if (Indexer.ruleMembershipExcludedIsMember(iindex) === false) throw Error('ruleMembershipExcludedIsMember');
// BR_G86
if ((await Indexer.ruleToBeKickedArePresent(iindex, dal)) === false) throw Error('ruleToBeKickedArePresent');
if ((await Indexer.ruleToBeKickedArePresent(iindex, dal)) === false) {
throw Error('ruleToBeKickedArePresent')
}
// BR_G103
if (Indexer.ruleTxWritability(sindex) === false) throw Error('ruleTxWritability');
// BR_G87
......@@ -174,7 +188,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
return { index, HEAD }
}
async pushTheBlock(obj: BlockDTO, index: IndexEntry[], HEAD: DBHead | null, conf: ConfDTO, dal: any, logger: any, trim = true) {
static async pushTheBlock(obj:BlockDTO, index:IndexEntry[], HEAD:DBHead | null, conf:ConfDTO, dal:FileDAL, logger:any, trim = true) {
const start = Date.now();
const block = BlockDTO.fromJSONObject(obj)
try {
......@@ -189,6 +203,13 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
}
logger.info('Block #' + block.number + ' added to the blockchain in %s ms', (Date.now() - start));
// Periodically, we trim the blockchain
if (block.number % CommonConstants.BLOCKS_COLLECT_THRESHOLD === 0) {
// Database trimming
await dal.loki.flushAndTrimData()
}
return BlockDTO.fromJSONObject(added)
}
catch(err) {
......@@ -201,7 +222,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// await supra.recordIndex(index)
}
async saveBlockData(current: DBBlock, block: BlockDTO, conf: ConfDTO, dal: any, logger: any, index: IndexEntry[], HEAD: DBHead | null, trim: boolean) {
static async saveBlockData(current:DBBlock|null, block:BlockDTO, conf:ConfDTO, dal:FileDAL, logger:any, index:IndexEntry[], HEAD:DBHead | null, trim: boolean) {
if (block.number == 0) {
await this.saveParametersForRoot(block, conf, dal);
}
......@@ -212,17 +233,14 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
await this.createNewcomers(indexes.iindex, dal, logger);
// Save indexes
await dal.bindexDAL.saveEntity(indexes.HEAD);
await dal.mindexDAL.insertBatch(indexes.mindex);
await dal.iindexDAL.insertBatch(indexes.iindex);
await dal.sindexDAL.insertBatch(indexes.sindex);
await dal.cindexDAL.insertBatch(indexes.cindex);
await dal.bindexDAL.insert(indexes.HEAD);
await dal.flushIndexes(indexes)
// Create/Update nodes in wotb
await this.updateMembers(block, dal);
// Update the wallets' blances
await this.updateWallets(indexes.sindex, dal)
await this.updateWallets(indexes.sindex, indexes.dividends, dal)
if (trim) {
const TAIL = await dal.bindexDAL.tail();
......@@ -237,6 +255,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
const MAX_BINDEX_SIZE = conf.forksize + bindexSize
const currentSize = indexes.HEAD.number - TAIL.number + 1
if (currentSize > MAX_BINDEX_SIZE) {
await dal.archiveBlocks()
await dal.trimIndexes(indexes.HEAD.number - MAX_BINDEX_SIZE);
}
}
......@@ -248,7 +267,7 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
await dal.updateWotbLinks(indexes.cindex);
// Create/Update certifications
await this.removeCertificationsFromSandbox(block, dal);
await DuniterBlockchain.removeCertificationsFromSandbox(block, dal);
// Create/Update memberships
await this.removeMembershipsFromSandbox(block, dal);
// Compute to be revoked members
......@@ -260,10 +279,12 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
// Saves the block (DAL)
await dal.saveBlock(dbb);
await dal.loki.commitData()
return dbb
}
async saveParametersForRoot(block:BlockDTO, conf:ConfDTO, dal:any) {
static async saveParametersForRoot(block:BlockDTO, conf:ConfDTO, dal:FileDAL) {
if (block.parameters) {
const bconf = BlockDTO.getConf(block)
conf.c = bconf.c;
......@@ -293,9 +314,10 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
}
}
async createNewcomers(iindex:IindexEntry[], dal:any, logger:any) {
for (const entry of iindex) {
if (entry.op == CommonConstants.IDX_CREATE) {
static async createNewcomers(iindex:IindexEntry[], dal:FileDAL, logger:any) {
for (const i of iindex) {
if (i.op == CommonConstants.IDX_CREATE) {
const entry = i as FullIindexEntry
// Reserves a wotb ID
entry.wotb_id = dal.wotb.addNode();
logger.trace('%s was affected wotb_id %s', entry.uid, entry.wotb_id);
......@@ -306,58 +328,64 @@ export class DuniterBlockchain extends MiscIndexedBlockchain {
}
}
async updateMembers(block:BlockDTO, dal:any) {
static async updateMembers(block:BlockDTO, dal:FileDAL) {
// Joiners (come back)
for (const inlineMS of block.joiners) {
let ms = MembershipDTO.fromInline(inlineMS)
const idty = await dal.getWrittenIdtyByPubkey(ms.issuer);
const idty = await dal.getWrittenIdtyByPubkeyForWotbID(ms.issuer);
dal.wotb.setEnabled(true, idty.wotb_id);
await dal.dividendDAL.setMember(true, ms.issuer)
}
// Revoked
for (const inlineRevocation of block.revoked) {
let revocation = RevocationDTO.fromInline(inlineRevocation)
await dal.revokeIdentity(revocation.pubkey, block.number);
await dal.revokeIdentity(revocation.pubkey)
}
// Excluded
for (const excluded of block.excluded) {
const idty = await dal.getWrittenIdtyByPubkey(excluded);
const idty = await dal.getWrittenIdtyByPubkeyForWotbID(excluded);
dal.wotb.setEnabled(false, idty.wotb_id);
await dal.dividendDAL.setMember(false, excluded)
}
}
async updateWallets(sindex:SindexEntry[], aDal:any, reverse = false) {
const differentConditions = _.uniq(sindex.map((entry) => entry.conditions))
static async updateWallets(sindex:SimpleTxEntryForWallet[], dividends:SimpleUdEntryForWallet[], aDal:any, reverse = false) {
const differentConditions = Underscore.uniq(sindex.map((entry) => entry.conditions).concat(dividends.map(d => d.conditions)))
for (const conditions of differentConditions) {
const creates = _.filter(sindex, (entry:SindexEntry) => entry.conditions === conditions && entry.op === CommonConstants.IDX_CREATE)
const updates = _.filter(sindex, (entry:SindexEntry) => entry.conditions === conditions && entry.op === CommonConstants.IDX_UPDATE)