Skip to content
Snippets Groups Projects
Commit 59149403 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

test: build_docker

parent 11c68005
Branches
No related tags found
No related merge requests found
Pipeline #40248 passed
...@@ -25,150 +25,521 @@ workflow: ...@@ -25,150 +25,521 @@ workflow:
.is_network_branch: &is_network_branch .is_network_branch: &is_network_branch
if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(network\/).+/ if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(network\/).+/
#
#.is_runtime_branch: &is_runtime_branch
# if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(runtime\/).+/
#
#sanity_tests:
# extends: .env
# rules:
# - if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TOPIC=="sanity-tests"'
# - when: never
# stage: schedule
# script:
# - cargo sanity-${CURRENCY}
#
#check_labels:
# extends: .env
# stage: labels
# rules:
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
# - when: never
# script:
# - ./scripts/check_labels.sh $CI_MERGE_REQUEST_LABELS $CI_MERGE_REQUEST_MILESTONE
#
#check_metadata:
# extends: .env
# stage: tests
# rules:
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
# - when: never
# script:
# - ./scripts/check_metadata.sh
#
#.env:
# image: paritytech/ci-linux:production
# tags:
# - kepler
#
#fmt_and_clippy:
# extends: .env
# rules:
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"'
# when: never
# - <<: *is_network_branch
# when: never
# - if: $CI_PIPELINE_SOURCE == "merge_request_event"
# when: always
# - when: manual
# stage: quality
# script:
# - cargo fmt -- --version
# - cargo fmt -- --check
# - cargo clippy -- -V
# - cargo clippy -Zgit=shallow-deps --features runtime-benchmarks --all --tests -- -D warnings
#
#run_benchmarks:
# extends: .env
# stage: tests
# rules:
# - <<: *is_network_branch
# when: never
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - <<: *is_network_branch
# when: never
# - if: $CI_COMMIT_TAG
# when: never
# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
# - when: manual
# script:
# - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks
# - target/release/duniter benchmark storage --chain=dev --mul=2 --state-version=1 --weight-path=./runtime/gdev/src/weights/
# - target/release/duniter benchmark overhead --chain=dev --wasm-execution=compiled --warmup=1 --repeat=100 --weight-path=./runtime/gdev/src/weights/
# - target/release/duniter benchmark pallet --chain=dev --steps=5 --repeat=2 --pallet="*" --extrinsic="*" --wasm-execution=compiled --output=./runtime/gdev/src/weights/
# - cargo build -Zgit=shallow-deps --release --features runtime-benchmarks # Check if autogenerated weights work
#
#gtest_build:
# stage: build
# extends: .env
# rules:
# - <<: *is_network_branch
# when: never
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - if: $CI_COMMIT_TAG
# when: never
# - <<: *is_network_branch
# when: never
# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
# - when: manual
# variables:
# DEBIAN_FRONTEND: noninteractive
# script:
# - cargo build -Zgit=shallow-deps --no-default-features --features gtest
#
#build_deb:
# stage: deploy
# extends: .env
# rules:
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - if: $CI_COMMIT_TAG
# when: never
# - <<: *is_network_branch
# needs: ["build_raw_specs"]
# when: always
# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
# when: manual
# - when: manual
# variables:
# DEBIAN_FRONTEND: noninteractive
# script:
# - cargo install cargo-deb
# - cargo build -Zgit=shallow-deps --release
# - cargo deb --no-build -p duniter
# artifacts:
# paths:
# - target/debian/duniter*.deb
#
#build_rpm:
# stage: deploy
# extends: .env
# rules:
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - if: $CI_COMMIT_TAG
# when: never
# - <<: *is_network_branch
# needs: ["build_raw_specs"]
# when: always
# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
# when: manual
# - when: manual
# script:
# - cargo install cargo-generate-rpm
# - cargo build -Zgit=shallow-deps --release
# - cargo generate-rpm -p node
# artifacts:
# paths:
# - target/generate-rpm/duniter*.rpm
#
#tests:
# stage: tests
# extends: .env
# rules:
# - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
# when: manual
# - if: $CI_COMMIT_TAG
# when: never
# - <<: *is_network_branch
# when: never
# - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
# - when: manual
# variables:
# DEBIAN_FRONTEND: noninteractive
# script:
# - cargo tu
# - cargo tf
# - cargo cucumber-build
# - cargo cucumber
#
#.network_branch_vars: &define_network_branch_vars
# - export NETWORK=$(echo $CI_COMMIT_BRANCH | sed -e "s/network\///g")
# - echo "NETWORK = $NETWORK"
# - export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1")
# - echo "RUNTIME = $RUNTIME"
# # srtool specific
# - export RUNTIME_DIR=runtime/$RUNTIME
# - echo "RUNTIME_DIR = $RUNTIME_DIR"
# # srtool specific
# - export PACKAGE=$RUNTIME-runtime
# - echo "PACKAGE = $PACKAGE"
# # GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes.
# - export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | head -n1 | sed "s/version = \"//g" | sed "s/\"//")
# - echo $CLIENT_VERSION
# - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
# - echo $RUNTIME_VERSION
# - export CLIENT_MILESTONE="client-$CLIENT_VERSION"
# - echo $CLIENT_MILESTONE
# - export NETWORK_RELEASE="$NETWORK"
# - echo $NETWORK_RELEASE
# - export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION"
# # Tags for Docker images
# - export IMAGE_NAME="duniter/duniter-v2s-$NETWORK"
# - echo $IMAGE_NAME
# - export MANIFEST=localhost/manifest-$IMAGE_NAME:$DOCKER_TAG
# - echo $MANIFEST
# # Files to be pushed in a release
# - export RELEASE_FILE_G1_DATA=release/genesis.json
# - echo $RELEASE_FILE_G1_DATA
# - export RELEASE_FILE_SPEC_CONFIG=release/${RUNTIME}.yaml
# - echo $RELEASE_FILE_SPEC_CONFIG
# - export RELEASE_FILE_SPEC=release/${RUNTIME}.json
# - echo $RELEASE_FILE_SPEC
# - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm
# - echo $RELEASE_FILE_WASM
# - export RELEASE_FILE_RAW_SPEC=release/${RUNTIME}-raw.json
# - echo $RELEASE_FILE_RAW_SPEC
# - export RELEASE_FILE_CLIENT_SPEC=release/gdev_client-specs.yaml
# - echo $RELEASE_FILE_CLIENT_SPEC
# - export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION
# - echo $CLIENT_RELEASE_NAME
#
#.release_runtime_vars: &define_release_runtime_vars
# - echo "RUNTIME = $RUNTIME"
# - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
# - echo $RUNTIME_VERSION
# - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION"
# - echo $RUNTIME_MILESTONE
# - export SRTOOL_OUTPUT=$CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json
# - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
# - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm
# - echo $RELEASE_FILE_WASM
# # srtool specific
# - export RUNTIME_DIR=runtime/$RUNTIME
# - echo "RUNTIME_DIR = $RUNTIME_DIR"
# # srtool specific
# - export PACKAGE=$RUNTIME-runtime
# - echo "PACKAGE = $PACKAGE"
#
#trigger_runtime_release:
# stage: build
# rules:
# - <<: *is_runtime_branch
# when: manual
# variables:
# RUNTIME: gdev
# script:
# - *define_release_runtime_vars
# - echo "Vérification de l'absence de la release $RUNTIME_MILESTONE"
# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE"
# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
#
#trigger_network_release:
# stage: build
# rules:
# - <<: *is_network_branch
# when: manual
# script:
# - *define_network_branch_vars
# - echo "Vérification de l'absence de la release $NETWORK"
# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK"
# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
#
#trigger_client_release:
# stage: build
# rules:
# - <<: *is_network_branch
# when: manual
# script:
# - *define_network_branch_vars
# - echo "Vérification de la présence de la release $NETWORK"
# - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK"
# - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && echo "Release présente" || (echo "Release absente" && exit 1)
#
#docker_deploy:
# stage: release
# needs: ["build_raw_specs"]
# rules:
# - <<: *is_network_branch
# - when: never
# before_script:
# - sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )"
# - podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io
# script:
# - *define_network_branch_vars
# - podman manifest rm "$MANIFEST" 2>/dev/null || true
# - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME .
# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG"
# - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest"
# after_script:
# - *define_network_branch_vars
# - echo $MANIFEST
# - podman manifest rm "$MANIFEST"
# tags:
# - podman
#
############### SRTOOL ##############
#
## The Network Runtime is only built when creating a network release (i.e.: genesis)
#build_network_runtime:
# stage: build
# needs: ["trigger_network_release"]
# rules:
# - <<: *is_network_branch
# image: paritytech/srtool:1.81.0-0.17.0
# variables:
# SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
# script:
# - *define_network_branch_vars
# - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
# - mkdir -p $CI_PROJECT_DIR/release
# # Copy sources to the expected directory of srtool
# - cp -R * /build/
# # Build the runtime
# - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT
# - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/
# artifacts:
# expire_in: never
# name: "runtime"
# paths:
# - $CI_PROJECT_DIR/release
# tags:
# - kepler
#
#build_gdev_runtime:
# stage: build
# needs: ["trigger_runtime_release"]
# rules:
# - <<: *is_runtime_branch
# image: paritytech/srtool:1.77.0-0.15.0
# variables:
# RUNTIME: gdev
# script:
# - *define_release_runtime_vars
# - mkdir -p $CI_PROJECT_DIR/release
# # Copy sources to the expected directory of srtool
# - cp -R * /build/
# # Build the runtime
# - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT
# - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/
# artifacts:
# expire_in: never
# name: "runtime"
# paths:
# - $CI_PROJECT_DIR/release
# tags:
# - kepler
#
############### SPECS ##############
#
#g1_data:
# stage: build
# needs: ["trigger_network_release"]
# rules:
# - <<: *is_network_branch
# image: h30x/py-g1-migrator # this image already has plyvel python requirement and dependency
# variables:
# DEBIAN_FRONTEND: noninteractive
# LEVELDB_PATH: /dump/duniter_default/data/leveldb
# script:
# - *define_network_branch_vars
# # Duniter 1.8.7 dump
# - mkdir /dump
# - cd /dump
# # Export file constructed using Duniter 1.8.7:
# # - bin/duniter sync g1.cgeek.fr --store-txs --nointeractive --mdb 1.8.7
# # - mkdir -p /tmp/backup-g1-duniter-1.8.7
# # - cp -R $HOME/.config/duniter/1.8.7/data /tmp/backup-g1-duniter-1.8.7
# # - tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
# # Then the file is uploaded to dl.cgeek.fr manually
# - curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz
# - tar xvzf g1-dump.tgz
# - rm g1-dump.tgz
# - mv tmp/backup-g1-duniter-1.8.7 duniter_default
# # py-g1-migrator conversion
# - git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 /py-g1-migrator
# - cd /py-g1-migrator
# # Export genesis file
# - ./main.py # ./output/genesis.json
# # Expore history files for squid
# - ./squid-block.py # ./output/block_hist.json
# - ./squid-cert.py # ./output/cert_hist.json
# - ./squid-tx.py # ./output/tx_hist.json
# # Make the exported file available for next jobs
# - mkdir -p $CI_PROJECT_DIR/release/
# - cp output/genesis.json $CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA
# - cp output/block_hist.json $CI_PROJECT_DIR/release/
# - cp output/cert_hist.json $CI_PROJECT_DIR/release/
# - cp output/tx_hist.json $CI_PROJECT_DIR/release/
# artifacts:
# expire_in: never
# paths:
# - $CI_PROJECT_DIR/release/
# tags:
# - kepler
#
#build_specs:
# stage: build
# needs: ["build_network_runtime", "g1_data"]
# rules:
# - <<: *is_network_branch
# image: rust:1-bullseye
# extends: .env
# variables:
# DEBIAN_FRONTEND: noninteractive
# script:
# - *define_network_branch_vars
# - export FEATURES="--features $RUNTIME --no-default-features"
# - echo "FEATURES = $FEATURES"
# - export WASM_FILE="$CI_PROJECT_DIR/$RELEASE_FILE_WASM"
# - echo "WASM_FILE = $WASM_FILE"
# - export DUNITER_GENESIS_DATA=$CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA
# - echo "DUNITER_GENESIS_DATA = $DUNITER_GENESIS_DATA"
# - apt-get update
# - apt-get install -y clang cmake protobuf-compiler
# # Build the spec file (including the G1 data), e.g.: "release/gdev.json"
# - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > $RELEASE_FILE_SPEC
# # Save spec configuration file for release
# - cp resources/${RUNTIME}.yaml $RELEASE_FILE_SPEC_CONFIG
# artifacts:
# expire_in: never
# name: "runtime"
# paths:
# - $CI_PROJECT_DIR/release
# tags:
# - kepler
#
#build_raw_specs:
# stage: build
# needs: ["trigger_client_release"]
# rules:
# - <<: *is_network_branch
# image: rust:1-bullseye
# script:
# - *define_network_branch_vars
# - export FEATURES="--features $RUNTIME --no-default-features"
# - echo "FEATURES = $FEATURES"
# - apt-get update
# - apt-get install -y clang cmake protobuf-compiler jq
# # Print chainspec to file
# - cargo xtask print-spec $NETWORK_RELEASE > ${RUNTIME}-printed.json
# # Merge client specs into chainspec file (to update bootnodes for example)
# # 1. Download yq to create a json client spec file
# - wget https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_arm64 -O yq
# - chmod +x ./yq
# # 2. YML -> JSON for the client specs
# - cat node/specs/${RUNTIME}_client-specs.yaml | ./yq --output-format json > node/specs/${RUNTIME}_client-specs.json
# # 3. Merge the client spec file into chainspec file and create the final spec file (e.g. gdev.json)
# - jq -s '.[0] * .[1]' node/specs/${RUNTIME}_client-specs.json ${RUNTIME}-printed.json > ${RUNTIME}.json
# # Produce raw spec file
# - mkdir -p $CI_PROJECT_DIR/release
# - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}.json --disable-default-bootnode --raw > $RELEASE_FILE_RAW_SPEC
# artifacts:
# expire_in: never
# name: "runtime"
# paths:
# - $CI_PROJECT_DIR/release
# tags:
# - kepler
#
############### RELEASE ##############
#
#create_network_release:
# stage: release
# needs: ["build_specs"]
# rules:
# - <<: *is_network_branch
# image: rust:1-bullseye
# variables:
# # Used by `release-network` command
# SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
# script:
# - *define_network_branch_vars
# # Release creation
# - cargo xtask release-network $NETWORK_RELEASE $CI_COMMIT_BRANCH
# # g1-data (initial data)
# - cargo xtask create-asset-link $NETWORK_RELEASE g1-data.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_G1_DATA
# # gdev.yaml (spec configuration)
# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC_CONFIG
# # initial runtime
# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
# # the result: gdev.json (genesis)
# - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC
# artifacts:
# expire_in: never
# paths:
# - $CI_PROJECT_DIR/release/
# tags:
# - kepler
#
#create_client_release:
# stage: release
# needs: ["build_raw_specs"]
# rules:
# - <<: *is_network_branch
# image: rust:1-bullseye
# script:
# - *define_network_branch_vars
# # Create the GitLab release page + tag and associate the milestone
# - cargo xtask release-client $CLIENT_RELEASE_NAME $CI_COMMIT_BRANCH $CLIENT_MILESTONE
# # Add the client assets
# - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC
# - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC
# artifacts:
# expire_in: never
# paths:
# - $CI_PROJECT_DIR/release/
# tags:
# - kepler
#
#create_runtime_release:
# stage: release
# needs: ["build_gdev_runtime"]
# rules:
# - <<: *is_runtime_branch
# image: rust:1-bullseye
# variables:
# RUNTIME: gdev
# script:
# - *define_release_runtime_vars
# # Create the GitLab release page + tag and associate the milestone.
# # Note: the release name = the release tag = runtime milesone ($RUNTIME_MILESTONE)
# - cargo xtask release-runtime $RUNTIME_MILESTONE $RUNTIME $CI_COMMIT_BRANCH $RUNTIME_MILESTONE
# - cargo xtask create-asset-link $RUNTIME_MILESTONE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
# # In the future: also build gtest and g1 runtimes and atach them
# artifacts:
# expire_in: never
# paths:
# - $CI_PROJECT_DIR/release/
# tags:
# - kepler
.is_runtime_branch: &is_runtime_branch build_docker:
if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(runtime\/).+/
sanity_tests:
extends: .env
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TOPIC=="sanity-tests"'
- when: never
stage: schedule
script:
- cargo sanity-${CURRENCY}
check_labels:
extends: .env
stage: labels
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- when: never
script:
- ./scripts/check_labels.sh $CI_MERGE_REQUEST_LABELS $CI_MERGE_REQUEST_MILESTONE
check_metadata:
extends: .env
stage: tests
rules:
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
- when: never
script:
- ./scripts/check_metadata.sh
.env:
image: paritytech/ci-linux:production
tags:
- kepler
fmt_and_clippy:
extends: .env
rules:
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual
- if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"'
when: never
- <<: *is_network_branch
when: never
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
when: always
- when: manual
stage: quality
script:
- cargo fmt -- --version
- cargo fmt -- --check
- cargo clippy -- -V
- cargo clippy -Zgit=shallow-deps --features runtime-benchmarks --all --tests -- -D warnings
run_benchmarks:
extends: .env
stage: tests
rules:
- <<: *is_network_branch
when: never
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual
- <<: *is_network_branch
when: never
- if: $CI_COMMIT_TAG
when: never
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
- when: manual
script:
- cargo build -Zgit=shallow-deps --release --features runtime-benchmarks
- target/release/duniter benchmark storage --chain=dev --mul=2 --state-version=1 --weight-path=./runtime/gdev/src/weights/
- target/release/duniter benchmark overhead --chain=dev --wasm-execution=compiled --warmup=1 --repeat=100 --weight-path=./runtime/gdev/src/weights/
- target/release/duniter benchmark pallet --chain=dev --steps=5 --repeat=2 --pallet="*" --extrinsic="*" --wasm-execution=compiled --output=./runtime/gdev/src/weights/
- cargo build -Zgit=shallow-deps --release --features runtime-benchmarks # Check if autogenerated weights work
gtest_build:
stage: build
extends: .env
rules:
- <<: *is_network_branch
when: never
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual
- if: $CI_COMMIT_TAG
when: never
- <<: *is_network_branch
when: never
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
- when: manual
variables:
DEBIAN_FRONTEND: noninteractive
script:
- cargo build -Zgit=shallow-deps --no-default-features --features gtest
build_deb:
stage: deploy stage: deploy
extends: .env
rules:
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual
- if: $CI_COMMIT_TAG
when: never
- <<: *is_network_branch
needs: ["build_raw_specs"]
when: always
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
when: manual
- when: manual
variables:
DEBIAN_FRONTEND: noninteractive
script:
- cargo install cargo-deb
- cargo build -Zgit=shallow-deps --release
- cargo deb --no-build -p duniter
artifacts:
paths:
- target/debian/duniter*.deb
build_rpm:
stage: deploy
extends: .env
rules:
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual
- if: $CI_COMMIT_TAG
when: never
- <<: *is_network_branch
needs: ["build_raw_specs"]
when: always
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
when: manual
- when: manual
script:
- cargo install cargo-generate-rpm
- cargo build -Zgit=shallow-deps --release
- cargo generate-rpm -p node
artifacts:
paths:
- target/generate-rpm/duniter*.rpm
tests:
stage: tests
extends: .env
rules: rules:
- if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - if: $CI_COMMIT_REF_NAME =~ /^wip*$/
when: manual when: manual
...@@ -178,362 +549,26 @@ tests: ...@@ -178,362 +549,26 @@ tests:
when: never when: never
- if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"'
- when: manual - when: manual
variables:
DEBIAN_FRONTEND: noninteractive
script:
- cargo tu
- cargo tf
- cargo cucumber-build
- cargo cucumber
.network_branch_vars: &define_network_branch_vars
- export NETWORK=$(echo $CI_COMMIT_BRANCH | sed -e "s/network\///g")
- echo "NETWORK = $NETWORK"
- export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1")
- echo "RUNTIME = $RUNTIME"
# srtool specific
- export RUNTIME_DIR=runtime/$RUNTIME
- echo "RUNTIME_DIR = $RUNTIME_DIR"
# srtool specific
- export PACKAGE=$RUNTIME-runtime
- echo "PACKAGE = $PACKAGE"
# GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes.
- export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | head -n1 | sed "s/version = \"//g" | sed "s/\"//")
- echo $CLIENT_VERSION
- export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
- echo $RUNTIME_VERSION
- export CLIENT_MILESTONE="client-$CLIENT_VERSION"
- echo $CLIENT_MILESTONE
- export NETWORK_RELEASE="$NETWORK"
- echo $NETWORK_RELEASE
- export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION"
# Tags for Docker images
- export IMAGE_NAME="duniter/duniter-v2s-$NETWORK"
- echo $IMAGE_NAME
- export MANIFEST=localhost/manifest-$IMAGE_NAME:$DOCKER_TAG
- echo $MANIFEST
# Files to be pushed in a release
- export RELEASE_FILE_G1_DATA=release/genesis.json
- echo $RELEASE_FILE_G1_DATA
- export RELEASE_FILE_SPEC_CONFIG=release/${RUNTIME}.yaml
- echo $RELEASE_FILE_SPEC_CONFIG
- export RELEASE_FILE_SPEC=release/${RUNTIME}.json
- echo $RELEASE_FILE_SPEC
- export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm
- echo $RELEASE_FILE_WASM
- export RELEASE_FILE_RAW_SPEC=release/${RUNTIME}-raw.json
- echo $RELEASE_FILE_RAW_SPEC
- export RELEASE_FILE_CLIENT_SPEC=release/gdev_client-specs.yaml
- echo $RELEASE_FILE_CLIENT_SPEC
- export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION
- echo $CLIENT_RELEASE_NAME
.release_runtime_vars: &define_release_runtime_vars
- echo "RUNTIME = $RUNTIME"
- export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
- echo $RUNTIME_VERSION
- export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION"
- echo $RUNTIME_MILESTONE
- export SRTOOL_OUTPUT=$CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json
- echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
- export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm
- echo $RELEASE_FILE_WASM
# srtool specific
- export RUNTIME_DIR=runtime/$RUNTIME
- echo "RUNTIME_DIR = $RUNTIME_DIR"
# srtool specific
- export PACKAGE=$RUNTIME-runtime
- echo "PACKAGE = $PACKAGE"
trigger_runtime_release:
stage: build
rules:
- <<: *is_runtime_branch
when: manual
variables:
RUNTIME: gdev
script:
- *define_release_runtime_vars
- echo "Vérification de l'absence de la release $RUNTIME_MILESTONE"
- echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE"
- curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
trigger_network_release:
stage: build
rules:
- <<: *is_network_branch
when: manual
script:
- *define_network_branch_vars
- echo "Vérification de l'absence de la release $NETWORK"
- echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK"
- curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
trigger_client_release:
stage: build
rules:
- <<: *is_network_branch
when: manual
script:
- *define_network_branch_vars
- echo "Vérification de la présence de la release $NETWORK"
- echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK"
- curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && echo "Release présente" || (echo "Release absente" && exit 1)
docker_deploy:
stage: release
needs: ["build_raw_specs"]
rules:
- <<: *is_network_branch
- when: never
before_script: before_script:
- sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )" - sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )"
- podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io - podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io
script:
- *define_network_branch_vars
- podman manifest rm "$MANIFEST" 2>/dev/null || true
- podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME .
- podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG"
- podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest"
after_script:
- *define_network_branch_vars
- echo $MANIFEST
- podman manifest rm "$MANIFEST"
tags:
- podman
############## SRTOOL ##############
# The Network Runtime is only built when creating a network release (i.e.: genesis)
build_network_runtime:
stage: build
needs: ["trigger_network_release"]
rules:
- <<: *is_network_branch
image: paritytech/srtool:1.81.0-0.17.0
variables:
SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
script:
- *define_network_branch_vars
- echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
- mkdir -p $CI_PROJECT_DIR/release
# Copy sources to the expected directory of srtool
- cp -R * /build/
# Build the runtime
- /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT
- mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/
artifacts:
expire_in: never
name: "runtime"
paths:
- $CI_PROJECT_DIR/release
tags:
- kepler
build_gdev_runtime:
stage: build
needs: ["trigger_runtime_release"]
rules:
- <<: *is_runtime_branch
image: paritytech/srtool:1.77.0-0.15.0
variables: variables:
RUNTIME: gdev RUNTIME: gdev
IMAGE_NAME: "duniter/duniter-v2s-gdev-800"
DOCKER_TAG: "$CI_COMMIT_SHORT_SHA"
MANIFEST: "localhost/manifest-$IMAGE_NAME:$DOCKER_TAG"
script: script:
- *define_release_runtime_vars - echo "RUNTIME = $RUNTIME"
- mkdir -p $CI_PROJECT_DIR/release - echo "DOCKER_TAG = $DOCKER_TAG"
# Copy sources to the expected directory of srtool - echo "IMAGE_NAME = $IMAGE_NAME"
- cp -R * /build/ - echo "MANIFEST = $MANIFEST"
# Build the runtime # - podman manifest rm "$MANIFEST" 2>/dev/null || true
- /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT # - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME .
- mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ # - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG"
artifacts: # - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest"
expire_in: never # after_script:
name: "runtime" # - *define_network_branch_vars
paths: # - echo $MANIFEST
- $CI_PROJECT_DIR/release # - podman manifest rm "$MANIFEST"
tags:
- kepler
############## SPECS ##############
g1_data:
stage: build
needs: ["trigger_network_release"]
rules:
- <<: *is_network_branch
image: h30x/py-g1-migrator # this image already has plyvel python requirement and dependency
variables:
DEBIAN_FRONTEND: noninteractive
LEVELDB_PATH: /dump/duniter_default/data/leveldb
script:
- *define_network_branch_vars
# Duniter 1.8.7 dump
- mkdir /dump
- cd /dump
# Export file constructed using Duniter 1.8.7:
# - bin/duniter sync g1.cgeek.fr --store-txs --nointeractive --mdb 1.8.7
# - mkdir -p /tmp/backup-g1-duniter-1.8.7
# - cp -R $HOME/.config/duniter/1.8.7/data /tmp/backup-g1-duniter-1.8.7
# - tar -cvzf /tmp/backup-g1-duniter-1.8.7.tgz /tmp/backup-g1-duniter-1.8.7
# Then the file is uploaded to dl.cgeek.fr manually
- curl https://dl.cgeek.fr/public/backup-g1-duniter-1.8.7.tgz -o g1-dump.tgz
- tar xvzf g1-dump.tgz
- rm g1-dump.tgz
- mv tmp/backup-g1-duniter-1.8.7 duniter_default
# py-g1-migrator conversion
- git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 /py-g1-migrator
- cd /py-g1-migrator
# Export genesis file
- ./main.py # ./output/genesis.json
# Expore history files for squid
- ./squid-block.py # ./output/block_hist.json
- ./squid-cert.py # ./output/cert_hist.json
- ./squid-tx.py # ./output/tx_hist.json
# Make the exported file available for next jobs
- mkdir -p $CI_PROJECT_DIR/release/
- cp output/genesis.json $CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA
- cp output/block_hist.json $CI_PROJECT_DIR/release/
- cp output/cert_hist.json $CI_PROJECT_DIR/release/
- cp output/tx_hist.json $CI_PROJECT_DIR/release/
artifacts:
expire_in: never
paths:
- $CI_PROJECT_DIR/release/
tags:
- kepler
build_specs:
stage: build
needs: ["build_network_runtime", "g1_data"]
rules:
- <<: *is_network_branch
image: rust:1-bullseye
extends: .env
variables:
DEBIAN_FRONTEND: noninteractive
script:
- *define_network_branch_vars
- export FEATURES="--features $RUNTIME --no-default-features"
- echo "FEATURES = $FEATURES"
- export WASM_FILE="$CI_PROJECT_DIR/$RELEASE_FILE_WASM"
- echo "WASM_FILE = $WASM_FILE"
- export DUNITER_GENESIS_DATA=$CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA
- echo "DUNITER_GENESIS_DATA = $DUNITER_GENESIS_DATA"
- apt-get update
- apt-get install -y clang cmake protobuf-compiler
# Build the spec file (including the G1 data), e.g.: "release/gdev.json"
- cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > $RELEASE_FILE_SPEC
# Save spec configuration file for release
- cp resources/${RUNTIME}.yaml $RELEASE_FILE_SPEC_CONFIG
artifacts:
expire_in: never
name: "runtime"
paths:
- $CI_PROJECT_DIR/release
tags:
- kepler
build_raw_specs:
stage: build
needs: ["trigger_client_release"]
rules:
- <<: *is_network_branch
image: rust:1-bullseye
script:
- *define_network_branch_vars
- export FEATURES="--features $RUNTIME --no-default-features"
- echo "FEATURES = $FEATURES"
- apt-get update
- apt-get install -y clang cmake protobuf-compiler jq
# Print chainspec to file
- cargo xtask print-spec $NETWORK_RELEASE > ${RUNTIME}-printed.json
# Merge client specs into chainspec file (to update bootnodes for example)
# 1. Download yq to create a json client spec file
- wget https://github.com/mikefarah/yq/releases/download/v4.44.6/yq_linux_arm64 -O yq
- chmod +x ./yq
# 2. YML -> JSON for the client specs
- cat node/specs/${RUNTIME}_client-specs.yaml | ./yq --output-format json > node/specs/${RUNTIME}_client-specs.json
# 3. Merge the client spec file into chainspec file and create the final spec file (e.g. gdev.json)
- jq -s '.[0] * .[1]' node/specs/${RUNTIME}_client-specs.json ${RUNTIME}-printed.json > ${RUNTIME}.json
# Produce raw spec file
- mkdir -p $CI_PROJECT_DIR/release
- cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}.json --disable-default-bootnode --raw > $RELEASE_FILE_RAW_SPEC
artifacts:
expire_in: never
name: "runtime"
paths:
- $CI_PROJECT_DIR/release
tags:
- kepler
############## RELEASE ##############
create_network_release:
stage: release
needs: ["build_specs"]
rules:
- <<: *is_network_branch
image: rust:1-bullseye
variables:
# Used by `release-network` command
SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
script:
- *define_network_branch_vars
# Release creation
- cargo xtask release-network $NETWORK_RELEASE $CI_COMMIT_BRANCH
# g1-data (initial data)
- cargo xtask create-asset-link $NETWORK_RELEASE g1-data.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_G1_DATA
# gdev.yaml (spec configuration)
- cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC_CONFIG
# initial runtime
- cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
# the result: gdev.json (genesis)
- cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC
artifacts:
expire_in: never
paths:
- $CI_PROJECT_DIR/release/
tags:
- kepler
create_client_release:
stage: release
needs: ["build_raw_specs"]
rules:
- <<: *is_network_branch
image: rust:1-bullseye
script:
- *define_network_branch_vars
# Create the GitLab release page + tag and associate the milestone
- cargo xtask release-client $CLIENT_RELEASE_NAME $CI_COMMIT_BRANCH $CLIENT_MILESTONE
# Add the client assets
- cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC
- cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC
artifacts:
expire_in: never
paths:
- $CI_PROJECT_DIR/release/
tags:
- kepler
create_runtime_release:
stage: release
needs: ["build_gdev_runtime"]
rules:
- <<: *is_runtime_branch
image: rust:1-bullseye
variables:
RUNTIME: gdev
script:
- *define_release_runtime_vars
# Create the GitLab release page + tag and associate the milestone.
# Note: the release name = the release tag = runtime milesone ($RUNTIME_MILESTONE)
- cargo xtask release-runtime $RUNTIME_MILESTONE $RUNTIME $CI_COMMIT_BRANCH $RUNTIME_MILESTONE
- cargo xtask create-asset-link $RUNTIME_MILESTONE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
# In the future: also build gtest and g1 runtimes and atach them
artifacts:
expire_in: never
paths:
- $CI_PROJECT_DIR/release/
tags: tags:
- kepler - podman
\ No newline at end of file
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment