From 0f671ac6d1a2f4a8e4be99b6de7b254d98f65ff1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Moreau?= <cem.moreau@gmail.com>
Date: Mon, 30 Dec 2024 16:14:25 +0100
Subject: [PATCH] Resolve "Create a release from master"
 (nodes/rust/duniter-v2s!297)

* fix: review

* #139: fix: re-enable Docker ARM build

* #239: feat: release du runtime from `runtime/*` branches

* #239: fix: clippy

* #269: fix: remove workaround from !239

* #239: feat: release du runtime from master

* #239: fix: create_client_release must not depend on runtime milestone

* #239: refact: rename `release_runtime` module to `gitlab`
---
 .gitlab-ci.yml                                | 119 ++++++++++++++----
 xtask/src/{release_runtime.rs => gitlab.rs}   |  31 ++++-
 .../create_asset_link.rs                      |   0
 .../create_network_release.rs                 |   0
 .../create_release.rs                         |   0
 .../get_changes.rs                            |   0
 .../{release_runtime => gitlab}/get_issues.rs |   0
 .../get_release.rs                            |   0
 xtask/src/main.rs                             |  28 ++++-
 9 files changed, 148 insertions(+), 30 deletions(-)
 rename xtask/src/{release_runtime.rs => gitlab.rs} (92%)
 rename xtask/src/{release_runtime => gitlab}/create_asset_link.rs (100%)
 rename xtask/src/{release_runtime => gitlab}/create_network_release.rs (100%)
 rename xtask/src/{release_runtime => gitlab}/create_release.rs (100%)
 rename xtask/src/{release_runtime => gitlab}/get_changes.rs (100%)
 rename xtask/src/{release_runtime => gitlab}/get_issues.rs (100%)
 rename xtask/src/{release_runtime => gitlab}/get_release.rs (100%)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 6fdbebf62..0f4f3f7b8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,6 +26,9 @@ workflow:
 .is_network_branch: &is_network_branch
   if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(network\/).+/
 
+.is_runtime_branch: &is_runtime_branch
+  if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_BRANCH =~ /^(runtime\/).+/
+
 sanity_tests:
   extends: .env
   rules:
@@ -181,15 +184,19 @@ tests:
   - echo "NETWORK = $NETWORK"
   - export RUNTIME=$(echo $NETWORK | grep -Po "gdev|gtest|g1")
   - echo "RUNTIME = $RUNTIME"
+  # srtool specific
+  - export RUNTIME_DIR=runtime/$RUNTIME
+  - echo "RUNTIME_DIR = $RUNTIME_DIR"
+  # srtool specific
   - export PACKAGE=$RUNTIME-runtime
   - echo "PACKAGE = $PACKAGE"
   # GitLab milestone : used for both GitLab and Docker releases. Milestone must match source code's runtime version to fetch the git changes for release notes.
   - export CLIENT_VERSION=$(cat node/Cargo.toml | grep version | sed "s/version = \"//g" | sed "s/\"//")
   - echo $CLIENT_VERSION
-  - export RUNTIME_VERSION=$(cat runtime/gdev/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
+  - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
   - echo $RUNTIME_VERSION
-  - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION"
-  - echo $RUNTIME_MILESTONE
+  - export CLIENT_MILESTONE="client-$CLIENT_VERSION"
+  - echo $CLIENT_MILESTONE
   - export NETWORK_RELEASE="$NETWORK"
   - echo $NETWORK_RELEASE
   - export DOCKER_TAG="$RUNTIME_VERSION-$CLIENT_VERSION"
@@ -214,6 +221,36 @@ tests:
   - export CLIENT_RELEASE_NAME=$RUNTIME-$RUNTIME_VERSION-$CLIENT_VERSION
   - echo $CLIENT_RELEASE_NAME
 
+.release_runtime_vars: &define_release_runtime_vars
+  - echo "RUNTIME = $RUNTIME"
+  - export RUNTIME_VERSION=$(cat runtime/$RUNTIME/src/lib.rs | grep "spec_version:" | sed "s/ *spec_version. //g" | sed "s/,//g")
+  - echo $RUNTIME_VERSION
+  - export RUNTIME_MILESTONE="runtime-$RUNTIME_VERSION"
+  - echo $RUNTIME_MILESTONE
+  - export SRTOOL_OUTPUT=$CI_PROJECT_DIR/release/srtool_output_$RUNTIME.json
+  - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
+  - export RELEASE_FILE_WASM=release/${RUNTIME}_runtime.compact.compressed.wasm
+  - echo $RELEASE_FILE_WASM
+  # srtool specific
+  - export RUNTIME_DIR=runtime/$RUNTIME
+  - echo "RUNTIME_DIR = $RUNTIME_DIR"
+  # srtool specific
+  - export PACKAGE=$RUNTIME-runtime
+  - echo "PACKAGE = $PACKAGE"
+
+trigger_runtime_release:
+  stage: build
+  rules:
+    - <<: *is_runtime_branch
+      when: manual
+  variables:
+    RUNTIME: gdev
+  script:
+    - *define_release_runtime_vars
+    - echo "Vérification de l'absence de la release $RUNTIME_MILESTONE"
+    - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE"
+    - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$RUNTIME_MILESTONE --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
+
 trigger_network_release:
   stage: build
   rules:
@@ -221,7 +258,7 @@ trigger_network_release:
       when: manual
   script:
     - *define_network_branch_vars
-    - echo "Vérification de la présence de la release $NETWORK"
+    - echo "Vérification de l'absence de la release $NETWORK"
     - echo "Contrôle de l'URL https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK"
     - curl -s https://git.duniter.org/api/v4/projects/$CI_PROJECT_ID/releases/$NETWORK --fail 1>/dev/null && (echo "Release déjà présente" && exit 1) || echo "Release absente"
 
@@ -248,7 +285,7 @@ docker_deploy:
   script:
     - *define_network_branch_vars
     - podman manifest rm "$MANIFEST" 2>/dev/null || true
-    - podman build --layers --platform linux/amd64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME .
+    - podman build --layers --platform linux/amd64,linux/arm64 --manifest "$MANIFEST" -f docker/Dockerfile --build-arg chain=$RUNTIME .
     - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:$DOCKER_TAG"
     - podman manifest push --all "$MANIFEST" "docker://docker.io/$IMAGE_NAME:latest"
   after_script:
@@ -260,23 +297,43 @@ docker_deploy:
 
 ############## SRTOOL ##############
 
-# We always build the runtime on a network/ branch, either it is for:
-# - creating a network release (i.e.: genesis)
-# - creating a client release (i.e.: which also includes the runtime)
-build_runtime:
+# The Network Runtime is only built when creating a network release (i.e.: genesis)
+build_network_runtime:
   stage: build
+  needs: ["trigger_network_release"]
   rules:
     - <<: *is_network_branch
   image: paritytech/srtool:1.77.0-0.15.0
   variables:
-    SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json
+    SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
   script:
     - *define_network_branch_vars
-    - export RUNTIME_DIR=runtime/$RUNTIME
-    - echo "RUNTIME_DIR = $RUNTIME_DIR"
     - echo "SRTOOL_OUTPUT = $SRTOOL_OUTPUT"
     - mkdir -p $CI_PROJECT_DIR/release
-    - echo -e "[toolchain]\nchannel = \"`curl -s https://raw.githubusercontent.com/paritytech/srtool/master/RUSTC_VERSION`\"\ncomponents = [ \"rust-std\", \"rust-src\" ]" > $RUNTIME_DIR/rust-toolchain.toml # Workaround see !239
+    # Copy sources to the expected directory of srtool
+    - cp -R * /build/
+    # Build the runtime
+    - /srtool/build --app --json -cM | tee -a $SRTOOL_OUTPUT
+    - mv /build/runtime/$RUNTIME/target/srtool/release/wbuild/$RUNTIME-runtime/${RUNTIME}_runtime.compact.compressed.wasm $CI_PROJECT_DIR/release/
+  artifacts:
+    expire_in: never
+    name: "runtime"
+    paths:
+      - $CI_PROJECT_DIR/release
+  tags:
+    - kepler
+
+build_gdev_runtime:
+  stage: build
+  needs: ["trigger_runtime_release"]
+  rules:
+    - <<: *is_runtime_branch
+  image: paritytech/srtool:1.77.0-0.15.0
+  variables:
+    RUNTIME: gdev
+  script:
+    - *define_release_runtime_vars
+    - mkdir -p $CI_PROJECT_DIR/release
     # Copy sources to the expected directory of srtool
     - cp -R * /build/
     # Build the runtime
@@ -340,7 +397,7 @@ g1_data:
 
 build_specs:
   stage: build
-  needs: ["build_runtime", "g1_data"]
+  needs: ["build_network_runtime", "g1_data"]
   rules:
     - <<: *is_network_branch
   extends: .env
@@ -403,7 +460,7 @@ create_network_release:
   image: rust:1-bullseye
   variables:
     # Used by `release-network` command
-    SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json
+    SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/network_srtool_output.json
   script:
     - *define_network_branch_vars
     # Release creation
@@ -425,17 +482,15 @@ create_network_release:
 
 create_client_release:
   stage: release
-  needs: ["build_runtime", "build_raw_specs"]
+  needs: ["build_raw_specs"]
   rules:
     - <<: *is_network_branch
   image: rust:1-bullseye
-  variables:
-    # Used by `release-runtime` command
-    SRTOOL_OUTPUT: $CI_PROJECT_DIR/release/srtool_output.json
   script:
     - *define_network_branch_vars
-    - cargo xtask release-runtime $CLIENT_RELEASE_NAME $NETWORK_RELEASE $CI_COMMIT_BRANCH $RUNTIME_MILESTONE
-    - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
+    # Create the GitLab release page + tag and associate the milestone
+    - cargo xtask release-client $CLIENT_RELEASE_NAME $CI_COMMIT_BRANCH $CLIENT_MILESTONE
+    # Add the client assets
     - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}_client-specs.yaml https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_CLIENT_SPEC
     - cargo xtask create-asset-link $CLIENT_RELEASE_NAME ${RUNTIME}-raw.json https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_RAW_SPEC
   artifacts:
@@ -444,3 +499,25 @@ create_client_release:
       - $CI_PROJECT_DIR/release/
   tags:
     - kepler
+
+create_runtime_release:
+  stage: release
+  needs: ["build_gdev_runtime"]
+  rules:
+    - <<: *is_runtime_branch
+  image: rust:1-bullseye
+  variables:
+    RUNTIME: gdev
+  script:
+    - *define_release_runtime_vars
+    # Create the GitLab release page + tag and associate the milestone.
+    # Note: the release name = the release tag = runtime milesone ($RUNTIME_MILESTONE)
+    - cargo xtask release-runtime $RUNTIME_MILESTONE $RUNTIME $CI_COMMIT_BRANCH $RUNTIME_MILESTONE
+    - cargo xtask create-asset-link $RUNTIME_MILESTONE ${RUNTIME}_runtime.compact.compressed.wasm https://nodes.pages.duniter.org/-/rust/duniter-v2s/-/jobs/$CI_JOB_ID/artifacts/$RELEASE_FILE_WASM
+    # In the future: also build gtest and g1 runtimes and atach them
+  artifacts:
+    expire_in: never
+    paths:
+      - $CI_PROJECT_DIR/release/
+  tags:
+    - kepler
diff --git a/xtask/src/release_runtime.rs b/xtask/src/gitlab.rs
similarity index 92%
rename from xtask/src/release_runtime.rs
rename to xtask/src/gitlab.rs
index 9f9d61548..5cbbf1d6d 100644
--- a/xtask/src/release_runtime.rs
+++ b/xtask/src/gitlab.rs
@@ -91,14 +91,37 @@ pub(super) async fn release_runtime(
     branch: String,
     milestone: String,
 ) -> Result<()> {
-    let mut release_notes = String::from(
+    release(
+        "Runtime".to_string(),
+        name,
+        Some(network),
+        branch,
+        milestone,
+    )
+    .await
+}
+
+pub(super) async fn release_client(name: String, branch: String, milestone: String) -> Result<()> {
+    release("Client".to_string(), name, None, branch, milestone).await
+}
+
+async fn release(
+    title: String,
+    name: String,
+    network: Option<String>,
+    branch: String,
+    milestone: String,
+) -> Result<()> {
+    let mut release_notes = format!(
         "
-# Runtime
+# {title}
 
-",
+"
     );
 
-    add_srtool_notes(network, &mut release_notes)?;
+    if let Some(network) = network {
+        add_srtool_notes(network.clone(), &mut release_notes)?;
+    }
 
     // Get changes (list of MRs) from gitlab API
     let changes = get_changes::get_changes(milestone.clone()).await?;
diff --git a/xtask/src/release_runtime/create_asset_link.rs b/xtask/src/gitlab/create_asset_link.rs
similarity index 100%
rename from xtask/src/release_runtime/create_asset_link.rs
rename to xtask/src/gitlab/create_asset_link.rs
diff --git a/xtask/src/release_runtime/create_network_release.rs b/xtask/src/gitlab/create_network_release.rs
similarity index 100%
rename from xtask/src/release_runtime/create_network_release.rs
rename to xtask/src/gitlab/create_network_release.rs
diff --git a/xtask/src/release_runtime/create_release.rs b/xtask/src/gitlab/create_release.rs
similarity index 100%
rename from xtask/src/release_runtime/create_release.rs
rename to xtask/src/gitlab/create_release.rs
diff --git a/xtask/src/release_runtime/get_changes.rs b/xtask/src/gitlab/get_changes.rs
similarity index 100%
rename from xtask/src/release_runtime/get_changes.rs
rename to xtask/src/gitlab/get_changes.rs
diff --git a/xtask/src/release_runtime/get_issues.rs b/xtask/src/gitlab/get_issues.rs
similarity index 100%
rename from xtask/src/release_runtime/get_issues.rs
rename to xtask/src/gitlab/get_issues.rs
diff --git a/xtask/src/release_runtime/get_release.rs b/xtask/src/gitlab/get_release.rs
similarity index 100%
rename from xtask/src/release_runtime/get_release.rs
rename to xtask/src/gitlab/get_release.rs
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index 07dd4e976..fd574f057 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -17,7 +17,7 @@
 #![feature(let_chains)]
 
 mod gen_doc;
-mod release_runtime;
+mod gitlab;
 
 use anyhow::{Context, Result};
 use clap::Parser;
@@ -57,9 +57,22 @@ enum DuniterXTaskCommand {
     ReleaseNetwork { network: String, branch: String },
     /// Release a new runtime
     ReleaseRuntime {
+        /// Name of the release + tag to be applied
         name: String,
+        /// Name of the network to be put in the release notes title of the srtool part
         network: String,
+        /// Branch on which the tag `name` will be created during the release
         branch: String,
+        /// Name of the milestone to add this release to
+        milestone: String,
+    },
+    /// Release a new client for a network
+    ReleaseClient {
+        /// Name of the release + tag to be applied
+        name: String,
+        /// Branch on which the tag `name` will be created during the release
+        branch: String,
+        /// Name of the milestone to add this release to
         milestone: String,
     },
     /// Print the chainSpec published on given Network Release
@@ -104,20 +117,25 @@ async fn main() -> Result<()> {
             inject_runtime_code(&raw_spec, &runtime)
         }
         DuniterXTaskCommand::ReleaseNetwork { network, branch } => {
-            release_runtime::release_network(network, branch).await
+            gitlab::release_network(network, branch).await
         }
         DuniterXTaskCommand::ReleaseRuntime {
             name,
             network,
             branch,
             milestone,
-        } => release_runtime::release_runtime(name, network, branch, milestone).await,
-        DuniterXTaskCommand::PrintSpec { network } => release_runtime::print_spec(network).await,
+        } => gitlab::release_runtime(name, network, branch, milestone).await,
+        DuniterXTaskCommand::ReleaseClient {
+            name,
+            branch,
+            milestone,
+        } => gitlab::release_client(name, branch, milestone).await,
+        DuniterXTaskCommand::PrintSpec { network } => gitlab::print_spec(network).await,
         DuniterXTaskCommand::CreateAssetLink {
             tag,
             asset_name,
             asset_url,
-        } => release_runtime::create_asset_link(tag, asset_name, asset_url).await,
+        } => gitlab::create_asset_link(tag, asset_name, asset_url).await,
         DuniterXTaskCommand::Test => test(),
     }
 }
-- 
GitLab