diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6e33b6b46e83503fc4e037e9c6c6e7ead8ef3c09..88dbca5aea438f2ae7df22de5b7daa7c25f896f4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -23,6 +23,9 @@ workflow: - Cargo.toml - Cargo.lock +.is_network_branch: &is_network_branch + if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(network\/).+/ + sanity_tests: extends: .env rules: @@ -62,7 +65,7 @@ fmt_and_clippy: when: manual - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH == "master"' when: never - - if: '$CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/' + - <<: *is_network_branch when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: always @@ -80,6 +83,8 @@ run_benchmarks: rules: - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ when: manual + - <<: *is_network_branch + when: never - if: $CI_COMMIT_TAG when: never - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' @@ -94,6 +99,8 @@ gtest_build: stage: build extends: .env rules: + - <<: *is_network_branch + when: never - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ when: manual - if: $CI_COMMIT_TAG @@ -115,7 +122,7 @@ build_deb: when: manual - if: $CI_COMMIT_TAG when: never - - if: $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch when: never - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - when: manual @@ -129,34 +136,6 @@ build_deb: paths: - target/debian/duniter*.deb -gdev_srtool_build: - stage: build - rules: - - if: $CI_COMMIT_REF_NAME =~ /^wip*$/ - when: manual - - if: $CI_COMMIT_TAG - when: never - - if: $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ - when: never - - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - - when: manual - image: paritytech/srtool:1.74.0-0.13.0 - variables: - PACKAGE: gdev-runtime - RUNTIME_DIR: runtime/gdev - SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output_gdev.json - script: - - echo "Building runtime for gdev" - - mkdir -p $CI_PROJECT_DIR/release - - echo -e "[toolchain]\nchannel = \"`curl -s https://raw.githubusercontent.com/paritytech/srtool/master/RUSTC_VERSION`\"\ncomponents = [ \"rust-std\", \"rust-src\" ]\ntargets = [ \"wasm32-unknown-unknown\" ]" > $RUNTIME_DIR/rust-toolchain.toml # Workaround see !239 - # Copy sources to the expected directory of srtool - - cp -R * /build/ - # Build the runtime - - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT - - mv /build/runtime/gdev/target/srtool/release/wbuild/gdev-runtime/gdev_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/ - tags: - - kepler - tests: stage: tests extends: .env @@ -165,7 +144,7 @@ tests: when: manual - if: $CI_COMMIT_TAG when: never - - if: $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch when: never - if: '$CI_MERGE_REQUEST_ID || $CI_COMMIT_BRANCH == "master"' - when: manual @@ -177,75 +156,105 @@ tests: - cargo cucumber-build - cargo cucumber -.deploy_docker_multiplatform: +.network_branch_vars: &define_network_branch_vars + - export NETWORK=$(echo $CI_COMMIT_BRANCH | sed -e "s/network\///g") + - echo "NETWORK = $NETWORK" + - export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1") + - echo "RUNTIME = $RUNTIME" + - export PACKAGE=$RUNTIME-runtime + - echo "PACKAGE = $PACKAGE" + # GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes. + - export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | sed "s/version = \"//g" | sed "s/\"//") + - echo $CLIENT_VERSION + - export RUNTIME_VERSION=$(cat runtime/gdev/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g") + - echo $RUNTIME_VERSION + - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION" + - echo $RUNTIME_MILESTONE + - export NETWORK_RELEASE="$NETWORK" + - echo $NETWORK_RELEASE + - export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION" + # Tags for Docker images + - export IMAGE_NAME="duniter/duniter-v2s-$NETWORK" + - echo $IMAGE_NAME + - export MANIFEST=localhost/manifest-$IMAGE_NAME:$DOCKER_TAG + - echo $MANIFEST + # Files to be pushed in a release + - export RELEASE_FILE_G1_DATA=release/genesis.json + - echo $RELEASE_FILE_G1_DATA + - export RELEASE_FILE_SPEC_CONFIG=release/${RUNTIME}.yaml + - echo $RELEASE_FILE_SPEC_CONFIG + - export RELEASE_FILE_SPEC=release/${RUNTIME}.json + - echo $RELEASE_FILE_SPEC + - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm + - echo $RELEASE_FILE_WASM + - export RELEASE_FILE_RAW_SPEC=release/${RUNTIME}-raw.json + - echo $RELEASE_FILE_RAW_SPEC + - export RELEASE_FILE_CLIENT_SPEC=release/gdev_client-specs.yaml + - echo $RELEASE_FILE_CLIENT_SPEC + - export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION + - echo $CLIENT_RELEASE_NAME + +trigger_network_release: + stage: build + rules: + - <<: *is_network_branch + when: manual + script: + - *define_network_branch_vars + - echo "Vérification de la présence de la release $NETWORK" + - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" + - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente" + +trigger_client_release: stage: build rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch when: manual - # changes: - # - node/specs/$CHAIN-raw.json + script: + - *define_network_branch_vars + - echo "Vérification de la présence de la release $NETWORK" + - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK" + - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && echo "Release présente" || (echo "Release absente" && exit 1) + +docker_deploy: + stage: release + needs: ["build_raw_specs"] + rules: + - <<: *is_network_branch - when: never before_script: - sh -c "[ -n '$DUNITERTEAM_PASSWD' ] || ( echo No access to environment variable 'DUNITERTEAM_PASSWD'; exit 1 )" - podman login -u "duniterteam" -p "$DUNITERTEAM_PASSWD" docker.io script: - - export MILESTONE=$(echo $CI_COMMIT_BRANCH | sed -e "s/release\///g") - - echo $MILESTONE - - export MANIFEST=localhost/manifest-$IMAGE_NAME:$MILESTONE - - echo $MANIFEST + - *define_network_branch_vars - podman manifest rm "$MANIFEST" 2>/dev/null || true - - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile $PODMAN_BUILD_OPTIONS . - - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$MILESTONE" + - podman build --layers --platform linux/amd64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME . + - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG" - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest" after_script: + - *define_network_branch_vars + - echo $MANIFEST - podman manifest rm "$MANIFEST" - variables: - IMAGE_NAME: "duniter/duniter-v2s-$CHAIN" - PODMAN_BUILD_OPTIONS: "--build-arg chain=$CHAIN" tags: - podman -gdev_docker_deploy: - extends: .deploy_docker_multiplatform - variables: - CHAIN: gdev - -gtest_docker_deploy: - extends: .deploy_docker_multiplatform - variables: - CHAIN: gtest - -readme_docker_release_tag: - stage: deploy_readme - rules: - - if: "$CI_COMMIT_TAG && $CI_COMMIT_TAG =~ /^(v|runtime-)[0-9].*/" - - when: never - image: - name: chko/docker-pushrm - entrypoint: ["/bin/sh", "-c", "/docker-pushrm"] - variables: - DOCKER_USER: duniterteam - DOCKER_PASS: "$DUNITERTEAM_PASSWD" - PUSHRM_SHORT: "Duniter v2 based on Substrate framework" - PUSHRM_TARGET: "docker.io/duniter/duniter-v2s" - PUSHRM_DEBUG: 1 - PUSHRM_FILE: "$CI_PROJECT_DIR/docker/README.md" - script: "/bin/true" - ############## SRTOOL ############## -.srtool: +# We always build the runtime on a network/ branch, either it is for: +# - creating a network release (i.e.: genesis) +# - creating a client release (i.e.: which also includes the runtime) +build_runtime: stage: build rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ - image: paritytech/srtool:1.74.0-0.13.0 + - <<: *is_network_branch + image: paritytech/srtool:1.77.0-0.15.0 variables: - PACKAGE: $RUNTIME-runtime - RUNTIME_DIR: runtime/$RUNTIME - SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json + SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json script: - - echo "Building runtime for $RUNTIME" - - echo $CI_COMMIT_BRANCH | sed -e "s/release\///g" + - *define_network_branch_vars + - export RUNTIME_DIR=runtime/$RUNTIME + - echo "RUNTIME_DIR = $RUNTIME_DIR" + - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT" - mkdir -p $CI_PROJECT_DIR/release - echo -e "[toolchain]\nchannel = \"`curl -s https://raw.githubusercontent.com/paritytech/srtool/master/RUSTC_VERSION`\"\ncomponents = [ \"rust-std\", \"rust-src\" ]" > $RUNTIME_DIR/rust-toolchain.toml # Workaround see !239 # Copy sources to the expected directory of srtool @@ -261,27 +270,19 @@ readme_docker_release_tag: tags: - kepler -gdev_srtool: - extends: .srtool - variables: - RUNTIME: gdev - -gtest_srtool: - extends: .srtool - variables: - RUNTIME: gtest - ############## SPECS ############## -create_g1_data: +g1_data: stage: build + needs: ["trigger_network_release"] rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch image: h30x/py-g1-migrator # this image already has plyvel python requirement and dependency variables: DEBIAN_FRONTEND: noninteractive LEVELDB_PATH: /dump/duniter_default/data/leveldb script: + - *define_network_branch_vars # Duniter 1.8.7 dump - mkdir /dump - cd /dump @@ -296,7 +297,7 @@ create_g1_data: - rm g1-dump.tgz - mv tmp/backup-g1-duniter-1.8.7 duniter_default # py-g1-migrator conversion - - git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 --branch hugo/docker /py-g1-migrator + - git clone https://git.duniter.org/tools/py-g1-migrator.git --depth 1 /py-g1-migrator - cd /py-g1-migrator # Export genesis file - ./main.py # ./output/genesis.json @@ -306,7 +307,7 @@ create_g1_data: - ./squid-tx.py # ./output/tx_hist.json # Make the exported file available for next jobs - mkdir -p $CI_PROJECT_DIR/release/ - - cp output/genesis.json $CI_PROJECT_DIR/release/ + - cp output/genesis.json $CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA - cp output/block_hist.json $CI_PROJECT_DIR/release/ - cp output/cert_hist.json $CI_PROJECT_DIR/release/ - cp output/tx_hist.json $CI_PROJECT_DIR/release/ @@ -317,19 +318,28 @@ create_g1_data: tags: - kepler -.build_specs: +build_specs: stage: build + needs: ["build_runtime", "g1_data"] rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch extends: .env variables: - WASM_FILE: $CI_PROJECT_DIR/release/${RUNTIME}_runtime.compact.compressed.wasm - DUNITER_GENESIS_DATA: $CI_PROJECT_DIR/release/genesis.json # py-g1-migrator outputs this file with `./main.py` DEBIAN_FRONTEND: noninteractive script: - - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > release/${RUNTIME}.json - - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=release/${RUNTIME}.json --disable-default-bootnode --raw > release/${RUNTIME}-raw.json - - cp node/specs/${RUNTIME}_client-specs.yaml release/ + - *define_network_branch_vars + - export FEATURES="--features $RUNTIME --no-default-features" + - echo "FEATURES = $FEATURES" + - export WASM_FILE="$CI_PROJECT_DIR/$RELEASE_FILE_WASM" + - echo "WASM_FILE = $WASM_FILE" + - export DUNITER_GENESIS_DATA=$CI_PROJECT_DIR/$RELEASE_FILE_G1_DATA + - echo "DUNITER_GENESIS_DATA = $DUNITER_GENESIS_DATA" + - apt-get update + - apt-get install -y clang cmake protobuf-compiler + # Build the spec file (including the G1 data), e.g.: "release/gdev.json" + - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}_live > $RELEASE_FILE_SPEC + # Save spec configuration file for release + - cp resources/${RUNTIME}.yaml $RELEASE_FILE_SPEC_CONFIG artifacts: expire_in: never name: "runtime" @@ -338,46 +348,54 @@ create_g1_data: tags: - kepler -gdev_specs: - extends: .build_specs - needs: - - gdev_srtool - - create_g1_data - variables: - RUNTIME: gdev - -gtest_specs: - extends: .build_specs - needs: - - gtest_srtool - - create_g1_data - variables: - RUNTIME: gtest - FEATURES: --features gtest --no-default-features +build_raw_specs: + stage: build + needs: ["trigger_client_release"] + rules: + - <<: *is_network_branch + image: rust:1-bullseye + script: + - *define_network_branch_vars + - export FEATURES="--features $RUNTIME --no-default-features" + - echo "FEATURES = $FEATURES" + - apt-get update + - apt-get install -y clang cmake protobuf-compiler + # Print chainspec to file + - cargo xtask print-spec $NETWORK_RELEASE > ${RUNTIME}.json + # Produce raw spec file + - mkdir -p $CI_PROJECT_DIR/release + - cargo run -Zgit=shallow-deps ${FEATURES} -- build-spec --chain=${RUNTIME}.json --disable-default-bootnode --raw > $RELEASE_FILE_RAW_SPEC + artifacts: + expire_in: never + name: "runtime" + paths: + - $CI_PROJECT_DIR/release + tags: + - kepler ############## RELEASE ############## -create_release: +create_network_release: stage: release + needs: ["build_specs"] rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ - needs: ["create_g1_data", "gdev_srtool", "gtest_srtool"] - when: manual + - <<: *is_network_branch image: rust:1-bullseye variables: - SRTOOL_OUTPUT_GDEV: $CI_PROJECT_DIR/release/srtool_output_gdev.json - SRTOOL_OUTPUT_GTEST: $CI_PROJECT_DIR/release/srtool_output_gtest.json - SRTOOL_OUTPUT_G1: $CI_PROJECT_DIR/release/srtool_output_g1.json + # Used by `release-network` command + SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json script: + - *define_network_branch_vars # Release creation - - export MILESTONE=$(echo $CI_COMMIT_BRANCH | sed -e "s/release\///g") - - cargo xtask release-runtime $MILESTONE $CI_COMMIT_BRANCH - # We always ship runtimes: this is both a proof and a convenience - - cargo xtask create-asset-link $MILESTONE genesis.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/genesis.json - - cargo xtask create-asset-link $MILESTONE gdev_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/gdev_runtime.compact.compressed.wasm - - cargo xtask create-asset-link $MILESTONE gtest_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/gtest_runtime.compact.compressed.wasm - - cargo xtask create-asset-link $MILESTONE gdev_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/gdev_client-specs.yaml - - cargo xtask create-asset-link $MILESTONE gtest_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/gtest_client-specs.yaml + - cargo xtask release-network $NETWORK_RELEASE $CI_COMMIT_BRANCH + # g1-data (initial data) + - cargo xtask create-asset-link $NETWORK_RELEASE g1-data.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_G1_DATA + # gdev.yaml (spec configuration) + - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC_CONFIG + # initial runtime + - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM + # the result: gdev.json (genesis) + - cargo xtask create-asset-link $NETWORK_RELEASE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_SPEC artifacts: expire_in: never paths: @@ -385,38 +403,24 @@ create_release: tags: - kepler -# ------ RELEASE: ADD SPECS ------ - -.release_specs: +create_client_release: stage: release + needs: ["build_runtime", "build_raw_specs"] rules: - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(release\/runtime-)[0-9].*/ + - <<: *is_network_branch image: rust:1-bullseye + variables: + # Used by `release-runtime` command + SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json script: - - export MILESTONE=$(echo $CI_COMMIT_BRANCH | sed -e "s/release\///g") - - cargo xtask create-asset-link $MILESTONE ${RUNTIME}.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/${RUNTIME}.json - - cargo xtask create-asset-link $MILESTONE ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/${RUNTIME}-raw.json - - cargo xtask create-asset-link $MILESTONE ${RUNTIME}-indexer.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/release/${RUNTIME}-indexer.json - - echo "Release Docker file..." + - *define_network_branch_vars + - cargo xtask release-runtime $CLIENT_RELEASE_NAME $NETWORK_RELEASE $CI_COMMIT_BRANCH $RUNTIME_MILESTONE + - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM + - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC + - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC artifacts: expire_in: never paths: - $CI_PROJECT_DIR/release/ tags: - kepler - -release_gdev_specs: - extends: .release_specs - needs: - - create_release - - gdev_specs - variables: - RUNTIME: gdev - -release_gtest_specs: - extends: .release_specs - needs: - - create_release - - gtest_specs - variables: - RUNTIME: gtest diff --git a/Cargo.lock b/Cargo.lock index 68351943dc4776f4f32cb4810f987d2889302804..65ec4ffb191f8589725c3f67a9f1cd16ee659309 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -904,7 +904,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.67", ] @@ -4326,6 +4326,24 @@ dependencies = [ "tokio-rustls 0.24.1", ] +[[package]] +name = "hyper-rustls" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +dependencies = [ + "futures-util", + "http 1.1.0", + "hyper 1.4.0", + "hyper-util", + "rustls 0.23.10", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower-service", + "webpki-roots 0.26.3", +] + [[package]] name = "hyper-util" version = "0.1.5" @@ -4812,7 +4830,7 @@ dependencies = [ "hyper 0.14.29", "jsonrpsee-types 0.22.5", "pin-project", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "thiserror", @@ -4840,7 +4858,7 @@ dependencies = [ "parking_lot 0.12.3", "pin-project", "rand", - "rustc-hash", + "rustc-hash 1.1.0", "serde", "serde_json", "thiserror", @@ -4857,7 +4875,7 @@ checksum = "1ccf93fc4a0bfe05d851d37d7c32b7f370fe94336b52a2f0efc5f1981895c2e5" dependencies = [ "async-trait", "hyper 0.14.29", - "hyper-rustls", + "hyper-rustls 0.24.2", "jsonrpsee-core 0.22.5", "jsonrpsee-types 0.22.5", "serde", @@ -8124,7 +8142,7 @@ dependencies = [ "pin-project-lite", "quinn-proto 0.9.6", "quinn-udp 0.3.2", - "rustc-hash", + "rustc-hash 1.1.0", "rustls 0.20.9", "thiserror", "tokio", @@ -8143,13 +8161,31 @@ dependencies = [ "pin-project-lite", "quinn-proto 0.10.6", "quinn-udp 0.4.1", - "rustc-hash", + "rustc-hash 1.1.0", "rustls 0.21.12", "thiserror", "tokio", "tracing", ] +[[package]] +name = "quinn" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +dependencies = [ + "bytes", + "pin-project-lite 0.2.14", + "quinn-proto 0.11.8", + "quinn-udp 0.5.4", + "rustc-hash 2.0.0", + "rustls 0.23.10", + "socket2 0.5.7", + "thiserror", + "tokio", + "tracing", +] + [[package]] name = "quinn-proto" version = "0.9.6" @@ -8159,7 +8195,7 @@ dependencies = [ "bytes", "rand", "ring 0.16.20", - "rustc-hash", + "rustc-hash 1.1.0", "rustls 0.20.9", "slab", "thiserror", @@ -8177,7 +8213,7 @@ dependencies = [ "bytes", "rand", "ring 0.16.20", - "rustc-hash", + "rustc-hash 1.1.0", "rustls 0.21.12", "slab", "thiserror", @@ -8185,6 +8221,23 @@ dependencies = [ "tracing", ] +[[package]] +name = "quinn-proto" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +dependencies = [ + "bytes", + "rand", + "ring 0.17.8", + "rustc-hash 2.0.0", + "rustls 0.23.10", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + [[package]] name = "quinn-udp" version = "0.3.2" @@ -8211,6 +8264,19 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2 0.5.7", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -8416,7 +8482,7 @@ checksum = "ad156d539c879b7a24a363a2016d77961786e71f48f2e2fc8302a92abd2429a6" dependencies = [ "hashbrown 0.13.2", "log", - "rustc-hash", + "rustc-hash 1.1.0", "slice-group-by", "smallvec", ] @@ -8484,7 +8550,8 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", + "hyper-rustls 0.27.3", "hyper-util", "ipnet", "js-sys", @@ -8492,17 +8559,23 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "pin-project-lite", + "pin-project-lite 0.2.14", + "quinn 0.11.5", + "rustls 0.23.10", + "rustls-pemfile 2.1.2", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-rustls 0.26.0", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", + "webpki-roots 0.26.3", "winreg 0.52.0", ] @@ -8636,6 +8709,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc-hex" version = "2.1.0" @@ -9613,7 +9692,7 @@ dependencies = [ "futures 0.3.30", "futures-timer", "hyper 0.14.29", - "hyper-rustls", + "hyper-rustls 0.24.2", "log", "num_cpus", "once_cell", @@ -9881,7 +9960,7 @@ dependencies = [ "parity-scale-codec", "parking_lot 0.12.3", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "sc-client-api", "sc-tracing-proc-macro", "serde", @@ -11332,7 +11411,7 @@ name = "sp-rpc" version = "26.0.0" source = "git+https://github.com/duniter/duniter-polkadot-sdk?branch=duniter-substrate-v1.14.0#bcc60f3e4170c3908689252242f40761270c9a51" dependencies = [ - "rustc-hash", + "rustc-hash 1.1.0", "serde", "sp-core", ] diff --git a/Cargo.toml b/Cargo.toml index ccafea0179677850c7ae08b55f68ca653c4b890e..648c2d7343fe27b5e1778f0264c9848d875a6ba0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ placeholder = { version = "1.1.4", default-features = false } getrandom = { version = "0.2.12", default-features = false } clap = { version = "4.5.3" } clap_complete = { version = "4.5.1" } -reqwest = { version = "0.12.0", default-features = false } +reqwest = { version = "0.12.0", default-features = false, features = ["rustls-tls"] } glob = { version = "0.3.1", default-features = false } convert_case = { version = "0.6.0", default-features = false } subweight-core = { version = "3.3.1", default-features = false } diff --git a/docker/Dockerfile b/docker/Dockerfile index ec6e346f940758c846ea34f6218bee2b7d4c6d93..de4215ee03149c605324709f97160353fd3fb626 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,10 +11,15 @@ FROM --platform=$BUILDPLATFORM rust:1-bullseye as build ARG BUILDPLATFORM ARG TARGETPLATFORM +# Debug +RUN echo "BUILDPLATFORM = $BUILDPLATFORM" +RUN echo "TARGETPLATFORM = $TARGETPLATFORM" + # We need the target arch triplet in both Debian and rust flavor RUN echo "DEBIAN_ARCH_TRIPLET='$(dpkg-architecture -A${TARGETPLATFORM#linux/} -qDEB_TARGET_MULTIARCH)'" >>/root/dynenv RUN . /root/dynenv && \ echo "RUST_ARCH_TRIPLET='$(echo "$DEBIAN_ARCH_TRIPLET" | sed -E 's/-linux-/-unknown&/')'" >>/root/dynenv +RUN cat /root/dynenv WORKDIR /root @@ -91,5 +96,9 @@ USER duniter # Intall COPY --from=build /root/build /usr/local/bin/ +COPY --from=build /root/dynenv /var/lib/duniter COPY docker/docker-entrypoint /usr/local/bin/ COPY docker/docker-distance-entrypoint /usr/local/bin/ + +# Debug +RUN cat /var/lib/duniter/dynenv diff --git a/xtask/res/create_network_release.gql b/xtask/res/create_network_release.gql new file mode 100644 index 0000000000000000000000000000000000000000..694afccf1ade6864ba8d0824573f1bdd41a7c936 --- /dev/null +++ b/xtask/res/create_network_release.gql @@ -0,0 +1,16 @@ +mutation CreateReleaseMutation($branch: String!, $description: String!, $network: String! $links: [ReleaseAssetLinkInput!]!) { + releaseCreate(input: { + clientMutationId: "duniter-v2s-xtask" + description: $description + milestones: [] + name: $network + projectPath: "nodes/rust/duniter-v2s" + ref: $branch + tagName: $network + assets: { + links: $links + } + }) { + errors + } +} diff --git a/xtask/res/create_release.gql b/xtask/res/create_release.gql index eb31aef63f847a521b198dabb1b392c6207ca932..2ba71180fd37442b413065d836201a7633139b44 100644 --- a/xtask/res/create_release.gql +++ b/xtask/res/create_release.gql @@ -1,12 +1,12 @@ -mutation CreateReleaseMutation($branch: String!, $description: String!, $milestone: String!, $links: [ReleaseAssetLinkInput!]!) { +mutation CreateReleaseMutation($name: String!, $branch: String!, $description: String!, $milestone: String!, $links: [ReleaseAssetLinkInput!]!) { releaseCreate(input: { clientMutationId: "duniter-v2s-xtask" description: $description milestones: [$milestone] - name: $milestone + name: $name projectPath: "nodes/rust/duniter-v2s" ref: $branch - tagName: $milestone + tagName: $name assets: { links: $links } diff --git a/xtask/res/get_release.gql b/xtask/res/get_release.gql index 9ca58b82c65c2a85df944fd36ec030905dde479a..7fb7ef667b8243fb2247c8d21774854ac79edd9a 100644 --- a/xtask/res/get_release.gql +++ b/xtask/res/get_release.gql @@ -1,6 +1,6 @@ -query GetReleaseOfProjectQuery($milestone: String!) { +query GetReleaseOfProjectQuery($tag: String!) { project(fullPath: "nodes/rust/duniter-v2s") { - release(tagName: $milestone) { + release(tagName: $tag) { id tagName assets { diff --git a/xtask/src/main.rs b/xtask/src/main.rs index d22c237fecee09ba3b523858bd9070e8dfd78013..07dd4e976bf2ea863422021b4c4f7745855cdb0f 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -53,10 +53,17 @@ enum DuniterXTaskCommand { /// Raw spec filepath raw_spec: PathBuf, }, + /// Release a new network + ReleaseNetwork { network: String, branch: String }, /// Release a new runtime - ReleaseRuntime { milestone: String, branch: String }, - /// Update raw specs locally with the files published on a Release - UpdateRawSpecs { milestone: String }, + ReleaseRuntime { + name: String, + network: String, + branch: String, + milestone: String, + }, + /// Print the chainSpec published on given Network Release + PrintSpec { network: String }, /// Create asset in a release CreateAssetLink { tag: String, @@ -81,8 +88,14 @@ async fn main() -> Result<()> { ); std::process::exit(1); } - Command::new("rustc").arg("--version").status()?; - Command::new("cargo").arg("--version").status()?; + + match &args.command { + DuniterXTaskCommand::PrintSpec { .. } => { /* no print */ } + _ => { + Command::new("rustc").arg("--version").status()?; + Command::new("cargo").arg("--version").status()?; + } + } match args.command { DuniterXTaskCommand::Build { production } => build(production), @@ -90,12 +103,16 @@ async fn main() -> Result<()> { DuniterXTaskCommand::InjectRuntimeCode { runtime, raw_spec } => { inject_runtime_code(&raw_spec, &runtime) } - DuniterXTaskCommand::ReleaseRuntime { milestone, branch } => { - release_runtime::release_runtime(milestone, branch).await - } - DuniterXTaskCommand::UpdateRawSpecs { milestone } => { - release_runtime::update_raw_specs(milestone).await + DuniterXTaskCommand::ReleaseNetwork { network, branch } => { + release_runtime::release_network(network, branch).await } + DuniterXTaskCommand::ReleaseRuntime { + name, + network, + branch, + milestone, + } => release_runtime::release_runtime(name, network, branch, milestone).await, + DuniterXTaskCommand::PrintSpec { network } => release_runtime::print_spec(network).await, DuniterXTaskCommand::CreateAssetLink { tag, asset_name, diff --git a/xtask/src/release_runtime.rs b/xtask/src/release_runtime.rs index 53bff4a97fc51d64ba767a87b2badaace649207f..9f9d61548e0e51f47c14ca5b76519a2a2b133940 100644 --- a/xtask/src/release_runtime.rs +++ b/xtask/src/release_runtime.rs @@ -15,6 +15,7 @@ // along with Duniter-v2S. If not, see <https://www.gnu.org/licenses/>. mod create_asset_link; +mod create_network_release; mod create_release; mod get_changes; mod get_issues; @@ -61,42 +62,43 @@ struct CoreVersion { //transaction_version: u32, } -pub(super) async fn release_runtime(milestone: String, branch: String) -> Result<()> { - // TODO: check spec_version in the code and bump if necessary (with a commit) - // TODO: create and push a git tag runtime-{spec_version} +pub(super) async fn release_network(network: String, branch: String) -> Result<()> { + let mut release_notes = String::from( + " +# Runtime + +", + ); + add_srtool_notes(network.clone(), &mut release_notes)?; + + println!("{}", release_notes); + let gitlab_token = + std::env::var("GITLAB_TOKEN").with_context(|| "missing env var GITLAB_TOKEN")?; + create_network_release::create_network_release( + gitlab_token, + branch, + network, + release_notes.to_string(), + ) + .await?; + Ok(()) +} + +pub(super) async fn release_runtime( + name: String, + network: String, + branch: String, + milestone: String, +) -> Result<()> { let mut release_notes = String::from( " -# Runtimes +# Runtime ", ); - // Generate release notes - let runtimes = vec![ - ("ĞDev", "SRTOOL_OUTPUT_GDEV"), - ("ĞTest", "SRTOOL_OUTPUT_GTEST"), - ("Ğ1", "SRTOOL_OUTPUT_G1"), - ]; - for (currency, env_var) in runtimes { - if let Ok(sr_tool_output_file) = std::env::var(env_var) { - let read = fs::read_to_string(sr_tool_output_file); - match read { - Ok(sr_tool_output) => { - release_notes.push_str( - gen_release_notes(currency.to_string(), sr_tool_output) - .with_context(|| { - format!("Fail to generate release notes for {}", currency) - })? - .as_str(), - ); - } - Err(e) => { - eprintln!("srtool JSON output could not be read ({}). Skipped.", e) - } - } - } - } + add_srtool_notes(network, &mut release_notes)?; // Get changes (list of MRs) from gitlab API let changes = get_changes::get_changes(milestone.clone()).await?; @@ -128,26 +130,59 @@ pub(super) async fn release_runtime(milestone: String, branch: String) -> Result println!("{}", release_notes); let gitlab_token = std::env::var("GITLAB_TOKEN").with_context(|| "missing env var GITLAB_TOKEN")?; - create_release::create_release(gitlab_token, branch, milestone, release_notes.to_string()) - .await?; + create_release::create_release( + gitlab_token, + name, + branch, + milestone, + release_notes.to_string(), + ) + .await?; + + Ok(()) +} + +fn add_srtool_notes(network: String, release_notes: &mut String) -> Result<()> { + // Generate release notes + let currency = network.clone(); + let env_var = "SRTOOL_OUTPUT".to_string(); + if let Ok(sr_tool_output_file) = std::env::var(env_var) { + let read = fs::read_to_string(sr_tool_output_file); + match read { + Ok(sr_tool_output) => { + release_notes.push_str( + gen_release_notes(currency.to_string(), sr_tool_output) + .with_context(|| { + format!("Fail to generate release notes for {}", currency) + })? + .as_str(), + ); + } + Err(e) => { + eprintln!("srtool JSON output could not be read ({}). Skipped.", e) + } + } + } Ok(()) } -pub(super) async fn update_raw_specs(milestone: String) -> Result<()> { - let specs = vec!["gdev-raw.json", "gtest-raw.json", "g1-raw.json"]; - println!("Fetching release info…"); - let assets = get_release::get_release(milestone).await?; - for spec in specs { - if let Some(gdev_raw_specs) = assets.iter().find(|asset| asset.ends_with(spec)) { - println!("Downloading {}…", spec); - let client = reqwest::Client::new(); - let res = client.get(gdev_raw_specs).send().await?; - let write_to = format!("./node/specs/{}", spec); - fs::write(write_to, res.bytes().await?)?; +pub(super) async fn print_spec(network: String) -> Result<()> { + let spec_file = match network.clone() { + network if network.starts_with("g1") => "g1.json", + network if network.starts_with("gtest") => "gtest.json", + network if network.starts_with("gdev") => "gdev.json", + _ => { + return Err(anyhow!("Invalid network")); } + }; + let assets = get_release::get_release(network).await?; + if let Some(gdev_spec) = assets.iter().find(|asset| asset.ends_with(spec_file)) { + let client = reqwest::Client::new(); + let res = client.get(gdev_spec).send().await?; + let spec = String::from_utf8(res.bytes().await?.to_vec())?; + println!("{}", spec); } - println!("Done."); Ok(()) } diff --git a/xtask/src/release_runtime/create_network_release.rs b/xtask/src/release_runtime/create_network_release.rs new file mode 100644 index 0000000000000000000000000000000000000000..6736a94caa27a8e7e78e9f2e1a3dc2e255de6850 --- /dev/null +++ b/xtask/src/release_runtime/create_network_release.rs @@ -0,0 +1,76 @@ +// Copyright 2021 Axiom-Team +// +// This file is part of Duniter-v2S. +// +// Duniter-v2S is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, version 3 of the License. +// +// Duniter-v2S is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with Duniter-v2S. If not, see <https://www.gnu.org/licenses/>. + +use anyhow::{anyhow, Result}; +use graphql_client::{GraphQLQuery, Response}; + +#[derive(GraphQLQuery)] +#[graphql( + schema_path = "res/schema.gql", + query_path = "res/create_network_release.gql", + response_derives = "Debug" +)] +pub struct CreateReleaseMutation; + +pub(super) async fn create_network_release( + gitlab_token: String, + branch: String, + network: String, + release_notes: String, +) -> Result<()> { + // this is the important line + let request_body = CreateReleaseMutation::build_query(create_release_mutation::Variables { + branch, + description: release_notes, + network, + links: vec![], + }); + + let client = reqwest::Client::new(); + let res = client + .post("https://git.duniter.org/api/graphql") + .header("PRIVATE-TOKEN", gitlab_token) + .json(&request_body) + .send() + .await?; + let response_body: Response<create_release_mutation::ResponseData> = res.json().await?; + + if let Some(data) = response_body.data { + if let Some(release_create) = data.release_create { + if release_create.errors.is_empty() { + Ok(()) + } else { + println!("{} errors:", release_create.errors.len()); + for error in release_create.errors { + println!("{}", error); + } + Err(anyhow!("Logic errors")) + } + } else if let Some(errors) = response_body.errors { + Err(anyhow!("Errors: {:?}", errors)) + } else { + Err(anyhow!("Invalid response: no release_create")) + } + } else if let Some(errors) = response_body.errors { + println!("{} errors:", errors.len()); + for error in errors { + println!("{}", error); + } + Err(anyhow!("GraphQL errors")) + } else { + Err(anyhow!("Invalid response: no data nor errors")) + } +} diff --git a/xtask/src/release_runtime/create_release.rs b/xtask/src/release_runtime/create_release.rs index bece83eecc048f4c05fc2056f70775e712ac6500..ed5e26d3b8dd3444a29bca8ebd5e7909c607d5e7 100644 --- a/xtask/src/release_runtime/create_release.rs +++ b/xtask/src/release_runtime/create_release.rs @@ -27,12 +27,14 @@ pub struct CreateReleaseMutation; pub(super) async fn create_release( gitlab_token: String, + name: String, branch: String, milestone: String, release_notes: String, ) -> Result<()> { // this is the important line let request_body = CreateReleaseMutation::build_query(create_release_mutation::Variables { + name, branch, description: release_notes, milestone, diff --git a/xtask/src/release_runtime/get_release.rs b/xtask/src/release_runtime/get_release.rs index cec00aba9eb8630d4035a7000a32ba6a2ed0557a..238a3504dd28d50eb5b6cfefc14e6c437f6b1612 100644 --- a/xtask/src/release_runtime/get_release.rs +++ b/xtask/src/release_runtime/get_release.rs @@ -25,12 +25,10 @@ use graphql_client::{GraphQLQuery, Response}; )] pub struct GetReleaseOfProjectQuery; -pub(super) async fn get_release(milestone: String) -> Result<Vec<String>> { +pub(super) async fn get_release(tag: String) -> Result<Vec<String>> { // this is the important line let request_body = - GetReleaseOfProjectQuery::build_query(get_release_of_project_query::Variables { - milestone, - }); + GetReleaseOfProjectQuery::build_query(get_release_of_project_query::Variables { tag }); let client = reqwest::Client::new(); let res = client