diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 50e7032058c3928e98e07da0f2bb7b300e44c083..fa450f856c19f88b4096cc8e9c5d5a61b957ca4b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,150 +25,521 @@ workflow: .is_network_branch: &is_network_branch if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(network\/).+/ +# +#.is_runtime_branch: &is_runtime_branch +# if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(runtime\/).+/ +# +#sanity_tests: +# extends: .env +# rules: +# - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TOPIC=="sanity-tests"' +# - when: never +# stage: schedule +# script: +# - cargo sanity-${CURRENCY} +# +#check_labels: +# extends: .env +# stage: labels +# rules: +# - if: $CI_PIPELINE_SOURCE == "merge_request_event" +# - when: never +# script: +# - ./scripts/check_labels.sh $CI_MERGE_REQUEST_LABELS $CI_MERGE_REQUEST_MILESTONE +# +#check_metadata: +# extends: .env +# stage: tests +# rules: +# - if: $CI_PIPELINE_SOURCE == "merge_request_event" +# - when: never +# script: +# - ./scripts/check_metadata.sh +# +#.env: +# image: paritytech/ci-linux:production +# tags: +# - kepler +# +#fmt_and_clippy: +# extends: .env +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"' +# when: never +# - <<: *is_network_branch +# when: never +# - if: $CI_PIPELINE_SOURCE == "merge_request_event" +# when: always +# - when: manual +# stage: quality +# script: +# - cargo fmt -- --version +# - cargo fmt -- --check +# - cargo clippy -- -V +# - cargo clippy -Zgit=shallow-deps --features runtime-benchmarks --all --tests -- -D warnings +# +#run_benchmarks: +# extends: .env +# stage: tests +# rules: +# - <<: *is_network_branch +# when: never +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - <<: *is_network_branch +# when: never +# - if: $CI_COMMIT_TAG +# when: never +# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' +# - when: manual +# script: +# - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks +# - target/release/duniter benchmark storage --chain=dev --mul=2 --state-version=1 --weight-path=./runtime/gdev/src/weights/ +# - target/release/duniter benchmark overhead --chain=dev --wasm-execution=compiled --warmup=1 --repeat=100 --weight-path=./runtime/gdev/src/weights/ +# - target/release/duniter benchmark pallet --chain=dev --steps=5 --repeat=2 --pallet="*" --extrinsic="*" --wasm-execution=compiled --output=./runtime/gdev/src/weights/ +# - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks # Check if autogenerated weights work +# +#gtest_build: +# stage: build +# extends: .env +# rules: +# - <<: *is_network_branch +# when: never +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - if: $CI_COMMIT_TAG +# when: never +# - <<: *is_network_branch +# when: never +# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' +# - when: manual +# variables: +# DEBIAN_FRONTEND: noninteractive +# script: +# - cargo build -Zgit=shallow-deps --no-default-features --features gtest +# +#build_deb: +# stage: deploy +# extends: .env +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - if: $CI_COMMIT_TAG +# when: never +# - <<: *is_network_branch +# needs: ["build_raw_specs"] +# when: always +# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' +# when: manual +# - when: manual +# variables: +# DEBIAN_FRONTEND: noninteractive +# script: +# - cargo install cargo-deb +# - cargo build -Zgit=shallow-deps --release +# - cargo deb --no-build -p duniter +# artifacts: +# paths: +# - target/debian/duniter*.deb +# +#build_rpm: +# stage: deploy +# extends: .env +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - if: $CI_COMMIT_TAG +# when: never +# - <<: *is_network_branch +# needs: ["build_raw_specs"] +# when: always +# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' +# when: manual +# - when: manual +# script: +# - cargo install cargo-generate-rpm +# - cargo build -Zgit=shallow-deps --release +# - cargo generate-rpm -p node +# artifacts: +# paths: +# - target/generate-rpm/duniter*.rpm +# +#tests: +# stage: tests +# extends: .env +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ +# when: manual +# - if: $CI_COMMIT_TAG +# when: never +# - <<: *is_network_branch +# when: never +# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' +# - when: manual +# variables: +# DEBIAN_FRONTEND: noninteractive +# script: +# - cargo tu +# - cargo tf +# - cargo cucumber-build +# - cargo cucumber +# +#.network_branch_vars: &define_network_branch_vars +# - export NETWORK=$(echo $CI_COMMIT_BRANCH | sed -e "s/network\///g") +# - echo "NETWORK = $NETWORK" +# - export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1") +# - echo "RUNTIME = $RUNTIME" +# # srtool specific +# - export RUNTIME_DIR=runtime/$RUNTIME +# - echo "RUNTIME_DIR = $RUNTIME_DIR" +# # srtool specific +# - export PACKAGE=$RUNTIME-runtime +# - echo "PACKAGE = $PACKAGE" +# # GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes. +# - export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | head -n1 | sed "s/version = \"//g" | sed "s/\"//") +# - echo $CLIENT_VERSION +# - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g") +# - echo $RUNTIME_VERSION +# - export CLIENT_MILESTONE="client-$CLIENT_VERSION" +# - echo $CLIENT_MILESTONE +# - export NETWORK_RELEASE="$NETWORK" +# - echo $NETWORK_RELEASE +# - export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION" +# # Tags for Docker images +# - export IMAGE_NAME="duniter/duniter-v2s-$NETWORK" +# - echo $IMAGE_NAME +# - export MANIFEST=localhost/manifest-$IMAGE_NAME:$DOCKER_TAG +# - echo $MANIFEST +# # Files to be pushed in a release +# - export RELEASE_FILE_G1_DATA=release/genesis.json +# - echo $RELEASE_FILE_G1_DATA +# - export RELEASE_FILE_SPEC_CONFIG=release/${RUNTIME}.yaml +# - echo $RELEASE_FILE_SPEC_CONFIG +# - export RELEASE_FILE_SPEC=release/${RUNTIME}.json +# - echo $RELEASE_FILE_SPEC +# - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm +# - echo $RELEASE_FILE_WASM +# - export RELEASE_FILE_RAW_SPEC=release/${RUNTIME}-raw.json +# - echo $RELEASE_FILE_RAW_SPEC +# - export RELEASE_FILE_CLIENT_SPEC=release/gdev_client-specs.yaml +# - echo $RELEASE_FILE_CLIENT_SPEC +# - export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION +# - echo $CLIENT_RELEASE_NAME +# +#.release_runtime_vars: &define_release_runtime_vars +# - echo "RUNTIME = $RUNTIME" +# - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g") +# - echo $RUNTIME_VERSION +# - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION" +# - echo $RUNTIME_MILESTONE +# - export SRTOOL_OUTPUT=$CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json +# - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT" +# - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm +# - echo $RELEASE_FILE_WASM +# # srtool specific +# - export RUNTIME_DIR=runtime/$RUNTIME +# - echo "RUNTIME_DIR = $RUNTIME_DIR" +# # srtool specific +# - export PACKAGE=$RUNTIME-runtime +# - echo "PACKAGE = $PACKAGE" +# +#trigger_runtime_release: +# stage: build +# rules: +# - <<: *is_runtime_branch +# when: manual +# variables: +# RUNTIME: gdev +# script: +# - *define_release_runtime_vars +# - echo "Vérification de l'absence de la release $RUNTIME_MILESTONE" +# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE" +# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente" +# +#trigger_network_release: +# stage: build +# rules: +# - <<: *is_network_branch +# when: manual +# script: +# - *define_network_branch_vars +# - echo "Vérification de l'absence de la release $NETWORK" +# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" +# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente" +# +#trigger_client_release: +# stage: build +# rules: +# - <<: *is_network_branch +# when: manual +# script: +# - *define_network_branch_vars +# - echo "Vérification de la présence de la release $NETWORK" +# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" +# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && echo "Release présente" || (echo "Release absente" && exit 1) +# +#docker_deploy: +# stage: release +# needs: ["build_raw_specs"] +# rules: +# - <<: *is_network_branch +# - when: never +# before_script: +# - sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )" +# - podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io +# script: +# - *define_network_branch_vars +# - podman manifest rm "$MANIFEST" 2>/dev/null || true +# - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME . +# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG" +# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest" +# after_script: +# - *define_network_branch_vars +# - echo $MANIFEST +# - podman manifest rm "$MANIFEST" +# tags: +# - podman +# +############### SRTOOL ############## +# +## The Network Runtime is only built when creating a network release (i.e.: genesis) +#build_network_runtime: +# stage: build +# needs: ["trigger_network_release"] +# rules: +# - <<: *is_network_branch +# image: paritytech/srtool:1.81.0-0.17.0 +# variables: +# SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json +# script: +# - *define_network_branch_vars +# - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT" +# - mkdir -p $CI_PROJECT_DIR/release +# # Copy sources to the expected directory of srtool +# - cp -R * /build/ +# # Build the runtime +# - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT +# - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ +# artifacts: +# expire_in: never +# name: "runtime" +# paths: +# - $CI_PROJECT_DIR/release +# tags: +# - kepler +# +#build_gdev_runtime: +# stage: build +# needs: ["trigger_runtime_release"] +# rules: +# - <<: *is_runtime_branch +# image: paritytech/srtool:1.77.0-0.15.0 +# variables: +# RUNTIME: gdev +# script: +# - *define_release_runtime_vars +# - mkdir -p $CI_PROJECT_DIR/release +# # Copy sources to the expected directory of srtool +# - cp -R * /build/ +# # Build the runtime +# - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT +# - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ +# artifacts: +# expire_in: never +# name: "runtime" +# paths: +# - $CI_PROJECT_DIR/release +# tags: +# - kepler +# +############### SPECS ############## +# +#g1_data: +# stage: build +# needs: ["trigger_network_release"] +# rules: +# - <<: *is_network_branch +# image: h30x/py-g1-migrator # this image already has plyvel python requirement and dependency +# variables: +# DEBIAN_FRONTEND: noninteractive +# LEVELDB_PATH: /dump/duniter_default/data/leveldb +# script: +# - *define_network_branch_vars +# # Duniter 1.8.7 dump +# - mkdir /dump +# - cd /dump +# # Export file constructed using Duniter 1.8.7: +# # - bin/duniter sync g1.cgeek.fr --store-txs --nointeractive --mdb 1.8.7 +# # - mkdir -p /tmp/backup-g1-duniter-1.8.7 +# # - cp -R $HOME/.config/duniter/1.8.7/data /tmp/backup-g1-duniter-1.8.7 +# # - tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7 +# # Then the file is uploaded to dl.cgeek.fr manually +# - curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz +# - tar xvzf g1-dump.tgz +# - rm g1-dump.tgz +# - mv tmp/backup-g1-duniter-1.8.7 duniter_default +# # py-g1-migrator conversion +# - git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 /py-g1-migrator +# - cd /py-g1-migrator +# # Export genesis file +# - ./main.py # ./output/genesis.json +# # Expore history files for squid +# - ./squid-block.py # ./output/block_hist.json +# - ./squid-cert.py # ./output/cert_hist.json +# - ./squid-tx.py # ./output/tx_hist.json +# # Make the exported file available for next jobs +# - mkdir -p $CI_PROJECT_DIR/release/ +# - cp output/genesis.json $CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA +# - cp output/block_hist.json $CI_PROJECT_DIR/release/ +# - cp output/cert_hist.json $CI_PROJECT_DIR/release/ +# - cp output/tx_hist.json $CI_PROJECT_DIR/release/ +# artifacts: +# expire_in: never +# paths: +# - $CI_PROJECT_DIR/release/ +# tags: +# - kepler +# +#build_specs: +# stage: build +# needs: ["build_network_runtime", "g1_data"] +# rules: +# - <<: *is_network_branch +# image: rust:1-bullseye +# extends: .env +# variables: +# DEBIAN_FRONTEND: noninteractive +# script: +# - *define_network_branch_vars +# - export FEATURES="--features $RUNTIME --no-default-features" +# - echo "FEATURES = $FEATURES" +# - export WASM_FILE="$CI_PROJECT_DIR/$RELEASE_FILE_WASM" +# - echo "WASM_FILE = $WASM_FILE" +# - export DUNITER_GENESIS_DATA=$CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA +# - echo "DUNITER_GENESIS_DATA = $DUNITER_GENESIS_DATA" +# - apt-get update +# - apt-get install -y clang cmake protobuf-compiler +# # Build the spec file (including the G1 data), e.g.: "release/gdev.json" +# - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > $RELEASE_FILE_SPEC +# # Save spec configuration file for release +# - cp resources/${RUNTIME}.yaml $RELEASE_FILE_SPEC_CONFIG +# artifacts: +# expire_in: never +# name: "runtime" +# paths: +# - $CI_PROJECT_DIR/release +# tags: +# - kepler +# +#build_raw_specs: +# stage: build +# needs: ["trigger_client_release"] +# rules: +# - <<: *is_network_branch +# image: rust:1-bullseye +# script: +# - *define_network_branch_vars +# - export FEATURES="--features $RUNTIME --no-default-features" +# - echo "FEATURES = $FEATURES" +# - apt-get update +# - apt-get install -y clang cmake protobuf-compiler jq +# # Print chainspec to file +# - cargo xtask print-spec $NETWORK_RELEASE > ${RUNTIME}-printed.json +# # Merge client specs into chainspec file (to update bootnodes for example) +# # 1. Download yq to create a json client spec file +# - wget https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_arm64 -O yq +# - chmod +x ./yq +# # 2. YML -> JSON for the client specs +# - cat node/specs/${RUNTIME}_client-specs.yaml | ./yq --output-format json > node/specs/${RUNTIME}_client-specs.json +# # 3. Merge the client spec file into chainspec file and create the final spec file (e.g. gdev.json) +# - jq -s '.[0] * .[1]' node/specs/${RUNTIME}_client-specs.json ${RUNTIME}-printed.json > ${RUNTIME}.json +# # Produce raw spec file +# - mkdir -p $CI_PROJECT_DIR/release +# - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}.json --disable-default-bootnode --raw > $RELEASE_FILE_RAW_SPEC +# artifacts: +# expire_in: never +# name: "runtime" +# paths: +# - $CI_PROJECT_DIR/release +# tags: +# - kepler +# +############### RELEASE ############## +# +#create_network_release: +# stage: release +# needs: ["build_specs"] +# rules: +# - <<: *is_network_branch +# image: rust:1-bullseye +# variables: +# # Used by `release-network` command +# SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json +# script: +# - *define_network_branch_vars +# # Release creation +# - cargo xtask release-network $NETWORK_RELEASE $CI_COMMIT_BRANCH +# # g1-data (initial data) +# - cargo xtask create-asset-link $NETWORK_RELEASE g1-data.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_G1_DATA +# # gdev.yaml (spec configuration) +# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC_CONFIG +# # initial runtime +# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM +# # the result: gdev.json (genesis) +# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC +# artifacts: +# expire_in: never +# paths: +# - $CI_PROJECT_DIR/release/ +# tags: +# - kepler +# +#create_client_release: +# stage: release +# needs: ["build_raw_specs"] +# rules: +# - <<: *is_network_branch +# image: rust:1-bullseye +# script: +# - *define_network_branch_vars +# # Create the GitLab release page + tag and associate the milestone +# - cargo xtask release-client $CLIENT_RELEASE_NAME $CI_COMMIT_BRANCH $CLIENT_MILESTONE +# # Add the client assets +# - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC +# - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC +# artifacts: +# expire_in: never +# paths: +# - $CI_PROJECT_DIR/release/ +# tags: +# - kepler +# +#create_runtime_release: +# stage: release +# needs: ["build_gdev_runtime"] +# rules: +# - <<: *is_runtime_branch +# image: rust:1-bullseye +# variables: +# RUNTIME: gdev +# script: +# - *define_release_runtime_vars +# # Create the GitLab release page + tag and associate the milestone. +# # Note: the release name = the release tag = runtime milesone ($RUNTIME_MILESTONE) +# - cargo xtask release-runtime $RUNTIME_MILESTONE $RUNTIME $CI_COMMIT_BRANCH $RUNTIME_MILESTONE +# - cargo xtask create-asset-link $RUNTIME_MILESTONE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM +# # In the future: also build gtest and g1 runtimes and atach them +# artifacts: +# expire_in: never +# paths: +# - $CI_PROJECT_DIR/release/ +# tags: +# - kepler -.is_runtime_branch: &is_runtime_branch - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(runtime\/).+/ - -sanity_tests: - extends: .env - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TOPIC=="sanity-tests"' - - when: never - stage: schedule - script: - - cargo sanity-${CURRENCY} - -check_labels: - extends: .env - stage: labels - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - when: never - script: - - ./scripts/check_labels.sh $CI_MERGE_REQUEST_LABELS $CI_MERGE_REQUEST_MILESTONE - -check_metadata: - extends: .env - stage: tests - rules: - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - - when: never - script: - - ./scripts/check_metadata.sh - -.env: - image: paritytech/ci-linux:production - tags: - - kepler - -fmt_and_clippy: - extends: .env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"' - when: never - - <<: *is_network_branch - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: always - - when: manual - stage: quality - script: - - cargo fmt -- --version - - cargo fmt -- --check - - cargo clippy -- -V - - cargo clippy -Zgit=shallow-deps --features runtime-benchmarks --all --tests -- -D warnings - -run_benchmarks: - extends: .env - stage: tests - rules: - - <<: *is_network_branch - when: never - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - <<: *is_network_branch - when: never - - if: $CI_COMMIT_TAG - when: never - - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - - when: manual - script: - - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks - - target/release/duniter benchmark storage --chain=dev --mul=2 --state-version=1 --weight-path=./runtime/gdev/src/weights/ - - target/release/duniter benchmark overhead --chain=dev --wasm-execution=compiled --warmup=1 --repeat=100 --weight-path=./runtime/gdev/src/weights/ - - target/release/duniter benchmark pallet --chain=dev --steps=5 --repeat=2 --pallet="*" --extrinsic="*" --wasm-execution=compiled --output=./runtime/gdev/src/weights/ - - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks # Check if autogenerated weights work - -gtest_build: - stage: build - extends: .env - rules: - - <<: *is_network_branch - when: never - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - if: $CI_COMMIT_TAG - when: never - - <<: *is_network_branch - when: never - - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - - when: manual - variables: - DEBIAN_FRONTEND: noninteractive - script: - - cargo build -Zgit=shallow-deps --no-default-features --features gtest - -build_deb: +build_docker: stage: deploy - extends: .env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - if: $CI_COMMIT_TAG - when: never - - <<: *is_network_branch - needs: ["build_raw_specs"] - when: always - - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - when: manual - - when: manual - variables: - DEBIAN_FRONTEND: noninteractive - script: - - cargo install cargo-deb - - cargo build -Zgit=shallow-deps --release - - cargo deb --no-build -p duniter - artifacts: - paths: - - target/debian/duniter*.deb - -build_rpm: - stage: deploy - extends: .env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - if: $CI_COMMIT_TAG - when: never - - <<: *is_network_branch - needs: ["build_raw_specs"] - when: always - - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - when: manual - - when: manual - script: - - cargo install cargo-generate-rpm - - cargo build -Zgit=shallow-deps --release - - cargo generate-rpm -p node - artifacts: - paths: - - target/generate-rpm/duniter*.rpm - -tests: - stage: tests - extends: .env rules: - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ when: manual @@ -178,362 +549,26 @@ tests: when: never - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - when: manual - variables: - DEBIAN_FRONTEND: noninteractive - script: - - cargo tu - - cargo tf - - cargo cucumber-build - - cargo cucumber - -.network_branch_vars: &define_network_branch_vars - - export NETWORK=$(echo $CI_COMMIT_BRANCH | sed -e "s/network\///g") - - echo "NETWORK = $NETWORK" - - export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1") - - echo "RUNTIME = $RUNTIME" - # srtool specific - - export RUNTIME_DIR=runtime/$RUNTIME - - echo "RUNTIME_DIR = $RUNTIME_DIR" - # srtool specific - - export PACKAGE=$RUNTIME-runtime - - echo "PACKAGE = $PACKAGE" - # GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes. - - export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | head -n1 | sed "s/version = \"//g" | sed "s/\"//") - - echo $CLIENT_VERSION - - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g") - - echo $RUNTIME_VERSION - - export CLIENT_MILESTONE="client-$CLIENT_VERSION" - - echo $CLIENT_MILESTONE - - export NETWORK_RELEASE="$NETWORK" - - echo $NETWORK_RELEASE - - export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION" - # Tags for Docker images - - export IMAGE_NAME="duniter/duniter-v2s-$NETWORK" - - echo $IMAGE_NAME - - export MANIFEST=localhost/manifest-$IMAGE_NAME:$DOCKER_TAG - - echo $MANIFEST - # Files to be pushed in a release - - export RELEASE_FILE_G1_DATA=release/genesis.json - - echo $RELEASE_FILE_G1_DATA - - export RELEASE_FILE_SPEC_CONFIG=release/${RUNTIME}.yaml - - echo $RELEASE_FILE_SPEC_CONFIG - - export RELEASE_FILE_SPEC=release/${RUNTIME}.json - - echo $RELEASE_FILE_SPEC - - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm - - echo $RELEASE_FILE_WASM - - export RELEASE_FILE_RAW_SPEC=release/${RUNTIME}-raw.json - - echo $RELEASE_FILE_RAW_SPEC - - export RELEASE_FILE_CLIENT_SPEC=release/gdev_client-specs.yaml - - echo $RELEASE_FILE_CLIENT_SPEC - - export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION - - echo $CLIENT_RELEASE_NAME - -.release_runtime_vars: &define_release_runtime_vars - - echo "RUNTIME = $RUNTIME" - - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g") - - echo $RUNTIME_VERSION - - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION" - - echo $RUNTIME_MILESTONE - - export SRTOOL_OUTPUT=$CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json - - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT" - - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm - - echo $RELEASE_FILE_WASM - # srtool specific - - export RUNTIME_DIR=runtime/$RUNTIME - - echo "RUNTIME_DIR = $RUNTIME_DIR" - # srtool specific - - export PACKAGE=$RUNTIME-runtime - - echo "PACKAGE = $PACKAGE" - -trigger_runtime_release: - stage: build - rules: - - <<: *is_runtime_branch - when: manual - variables: - RUNTIME: gdev - script: - - *define_release_runtime_vars - - echo "Vérification de l'absence de la release $RUNTIME_MILESTONE" - - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE" - - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente" - -trigger_network_release: - stage: build - rules: - - <<: *is_network_branch - when: manual - script: - - *define_network_branch_vars - - echo "Vérification de l'absence de la release $NETWORK" - - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" - - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente" - -trigger_client_release: - stage: build - rules: - - <<: *is_network_branch - when: manual - script: - - *define_network_branch_vars - - echo "Vérification de la présence de la release $NETWORK" - - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" - - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && echo "Release présente" || (echo "Release absente" && exit 1) - -docker_deploy: - stage: release - needs: ["build_raw_specs"] - rules: - - <<: *is_network_branch - - when: never before_script: - sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )" - podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io - script: - - *define_network_branch_vars - - podman manifest rm "$MANIFEST" 2>/dev/null || true - - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME . - - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG" - - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest" - after_script: - - *define_network_branch_vars - - echo $MANIFEST - - podman manifest rm "$MANIFEST" - tags: - - podman - -############## SRTOOL ############## - -# The Network Runtime is only built when creating a network release (i.e.: genesis) -build_network_runtime: - stage: build - needs: ["trigger_network_release"] - rules: - - <<: *is_network_branch - image: paritytech/srtool:1.81.0-0.17.0 - variables: - SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json - script: - - *define_network_branch_vars - - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT" - - mkdir -p $CI_PROJECT_DIR/release - # Copy sources to the expected directory of srtool - - cp -R * /build/ - # Build the runtime - - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT - - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ - artifacts: - expire_in: never - name: "runtime" - paths: - - $CI_PROJECT_DIR/release - tags: - - kepler - -build_gdev_runtime: - stage: build - needs: ["trigger_runtime_release"] - rules: - - <<: *is_runtime_branch - image: paritytech/srtool:1.77.0-0.15.0 - variables: - RUNTIME: gdev - script: - - *define_release_runtime_vars - - mkdir -p $CI_PROJECT_DIR/release - # Copy sources to the expected directory of srtool - - cp -R * /build/ - # Build the runtime - - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT - - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ - artifacts: - expire_in: never - name: "runtime" - paths: - - $CI_PROJECT_DIR/release - tags: - - kepler - -############## SPECS ############## - -g1_data: - stage: build - needs: ["trigger_network_release"] - rules: - - <<: *is_network_branch - image: h30x/py-g1-migrator # this image already has plyvel python requirement and dependency - variables: - DEBIAN_FRONTEND: noninteractive - LEVELDB_PATH: /dump/duniter_default/data/leveldb - script: - - *define_network_branch_vars - # Duniter 1.8.7 dump - - mkdir /dump - - cd /dump - # Export file constructed using Duniter 1.8.7: - # - bin/duniter sync g1.cgeek.fr --store-txs --nointeractive --mdb 1.8.7 - # - mkdir -p /tmp/backup-g1-duniter-1.8.7 - # - cp -R $HOME/.config/duniter/1.8.7/data /tmp/backup-g1-duniter-1.8.7 - # - tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7 - # Then the file is uploaded to dl.cgeek.fr manually - - curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz - - tar xvzf g1-dump.tgz - - rm g1-dump.tgz - - mv tmp/backup-g1-duniter-1.8.7 duniter_default - # py-g1-migrator conversion - - git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 /py-g1-migrator - - cd /py-g1-migrator - # Export genesis file - - ./main.py # ./output/genesis.json - # Expore history files for squid - - ./squid-block.py # ./output/block_hist.json - - ./squid-cert.py # ./output/cert_hist.json - - ./squid-tx.py # ./output/tx_hist.json - # Make the exported file available for next jobs - - mkdir -p $CI_PROJECT_DIR/release/ - - cp output/genesis.json $CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA - - cp output/block_hist.json $CI_PROJECT_DIR/release/ - - cp output/cert_hist.json $CI_PROJECT_DIR/release/ - - cp output/tx_hist.json $CI_PROJECT_DIR/release/ - artifacts: - expire_in: never - paths: - - $CI_PROJECT_DIR/release/ - tags: - - kepler - -build_specs: - stage: build - needs: ["build_network_runtime", "g1_data"] - rules: - - <<: *is_network_branch - image: rust:1-bullseye - extends: .env - variables: - DEBIAN_FRONTEND: noninteractive - script: - - *define_network_branch_vars - - export FEATURES="--features $RUNTIME --no-default-features" - - echo "FEATURES = $FEATURES" - - export WASM_FILE="$CI_PROJECT_DIR/$RELEASE_FILE_WASM" - - echo "WASM_FILE = $WASM_FILE" - - export DUNITER_GENESIS_DATA=$CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA - - echo "DUNITER_GENESIS_DATA = $DUNITER_GENESIS_DATA" - - apt-get update - - apt-get install -y clang cmake protobuf-compiler - # Build the spec file (including the G1 data), e.g.: "release/gdev.json" - - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > $RELEASE_FILE_SPEC - # Save spec configuration file for release - - cp resources/${RUNTIME}.yaml $RELEASE_FILE_SPEC_CONFIG - artifacts: - expire_in: never - name: "runtime" - paths: - - $CI_PROJECT_DIR/release - tags: - - kepler - -build_raw_specs: - stage: build - needs: ["trigger_client_release"] - rules: - - <<: *is_network_branch - image: rust:1-bullseye - script: - - *define_network_branch_vars - - export FEATURES="--features $RUNTIME --no-default-features" - - echo "FEATURES = $FEATURES" - - apt-get update - - apt-get install -y clang cmake protobuf-compiler jq - # Print chainspec to file - - cargo xtask print-spec $NETWORK_RELEASE > ${RUNTIME}-printed.json - # Merge client specs into chainspec file (to update bootnodes for example) - # 1. Download yq to create a json client spec file - - wget https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_arm64 -O yq - - chmod +x ./yq - # 2. YML -> JSON for the client specs - - cat node/specs/${RUNTIME}_client-specs.yaml | ./yq --output-format json > node/specs/${RUNTIME}_client-specs.json - # 3. Merge the client spec file into chainspec file and create the final spec file (e.g. gdev.json) - - jq -s '.[0] * .[1]' node/specs/${RUNTIME}_client-specs.json ${RUNTIME}-printed.json > ${RUNTIME}.json - # Produce raw spec file - - mkdir -p $CI_PROJECT_DIR/release - - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}.json --disable-default-bootnode --raw > $RELEASE_FILE_RAW_SPEC - artifacts: - expire_in: never - name: "runtime" - paths: - - $CI_PROJECT_DIR/release - tags: - - kepler - -############## RELEASE ############## - -create_network_release: - stage: release - needs: ["build_specs"] - rules: - - <<: *is_network_branch - image: rust:1-bullseye - variables: - # Used by `release-network` command - SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json - script: - - *define_network_branch_vars - # Release creation - - cargo xtask release-network $NETWORK_RELEASE $CI_COMMIT_BRANCH - # g1-data (initial data) - - cargo xtask create-asset-link $NETWORK_RELEASE g1-data.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_G1_DATA - # gdev.yaml (spec configuration) - - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC_CONFIG - # initial runtime - - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM - # the result: gdev.json (genesis) - - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC - artifacts: - expire_in: never - paths: - - $CI_PROJECT_DIR/release/ - tags: - - kepler - -create_client_release: - stage: release - needs: ["build_raw_specs"] - rules: - - <<: *is_network_branch - image: rust:1-bullseye - script: - - *define_network_branch_vars - # Create the GitLab release page + tag and associate the milestone - - cargo xtask release-client $CLIENT_RELEASE_NAME $CI_COMMIT_BRANCH $CLIENT_MILESTONE - # Add the client assets - - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC - - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC - artifacts: - expire_in: never - paths: - - $CI_PROJECT_DIR/release/ - tags: - - kepler - -create_runtime_release: - stage: release - needs: ["build_gdev_runtime"] - rules: - - <<: *is_runtime_branch - image: rust:1-bullseye variables: RUNTIME: gdev + IMAGE_NAME: "duniter/duniter-v2s-gdev-800" + DOCKER_TAG: "$CI_COMMIT_SHORT_SHA" + MANIFEST: "localhost/manifest-$IMAGE_NAME:$DOCKER_TAG" script: - - *define_release_runtime_vars - # Create the GitLab release page + tag and associate the milestone. - # Note: the release name = the release tag = runtime milesone ($RUNTIME_MILESTONE) - - cargo xtask release-runtime $RUNTIME_MILESTONE $RUNTIME $CI_COMMIT_BRANCH $RUNTIME_MILESTONE - - cargo xtask create-asset-link $RUNTIME_MILESTONE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM - # In the future: also build gtest and g1 runtimes and atach them - artifacts: - expire_in: never - paths: - - $CI_PROJECT_DIR/release/ + - echo "RUNTIME = $RUNTIME" + - echo "DOCKER_TAG = $DOCKER_TAG" + - echo "IMAGE_NAME = $IMAGE_NAME" + - echo "MANIFEST = $MANIFEST" +# - podman manifest rm "$MANIFEST" 2>/dev/null || true +# - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME . +# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG" +# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest" +# after_script: +# - *define_network_branch_vars +# - echo $MANIFEST +# - podman manifest rm "$MANIFEST" tags: - - kepler + - podman \ No newline at end of file