From 934fddff5f8d42c489e3063e1f4a7e7ebc5206d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Veyret?= <stephane.veyret@neptura.org>
Date: Sun, 21 Jan 2018 11:22:13 +0100
Subject: [PATCH 1/5] Synchonize GitHub only on main repo
---
.gitlab-ci.yml | 36 +++++++++++++++++++-----------------
1 file changed, 19 insertions(+), 17 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c53f53871..e898a11c1 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,23 +4,26 @@ stages:
- test
- releases
- releases-page
+
push_to_github:
- stage: github-sync
- variables:
- GIT_STRATEGY: none
- tags:
- - redshift
- script:
- - rm -rf ./*
- - rm -rf .git
- - git clone --mirror $CI_REPOSITORY_URL .
- - git remote add github $GITHUB_URL_AND_KEY
- - git config --global user.email "contact@duniter.org"
- - git config --global user.name "Duniter"
- # Job would fail if we don't remove refs about pull requests
- - bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
- - mv packed-refs-new packed-refs
- - bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
+ stage: github-sync
+ variables:
+ GIT_STRATEGY: none
+ tags:
+ - redshift
+ script:
+ - rm -rf ./*
+ - rm -rf .git
+ - git clone --mirror $CI_REPOSITORY_URL .
+ - git remote add github $GITHUB_URL_AND_KEY
+ - git config --global user.email "contact@duniter.org"
+ - git config --global user.name "Duniter"
+ # Job would fail if we don't remove refs about pull requests
+ - bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
+ - mv packed-refs-new packed-refs
+ - bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
+ only:
+ - nodes/typescript/duniter
build:
stage: build
@@ -61,7 +64,6 @@ releases:test:
when: manual
except:
- tags
-
releases:
stage: releases
--
GitLab
From f782cebe57467f24ff7caa33de7d2973e68b431b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Veyret?= <stephane.veyret@neptura.org>
Date: Sun, 21 Jan 2018 11:51:01 +0100
Subject: [PATCH 2/5] Simplify gitlab-ci file
---
.gitlab-ci.yml | 57 ++++++++++++++++++++++----------------------------
1 file changed, 25 insertions(+), 32 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index e898a11c1..19c14499e 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -24,67 +24,61 @@ push_to_github:
- bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
only:
- nodes/typescript/duniter
-
-build:
- stage: build
+
+.nvm_env: &nvm_env
tags:
- redshift
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
+
+build:
+ <<: *nvm_env
+ stage: build
script:
- yarn
test:
+ <<: *nvm_env
stage: test
- tags:
- - redshift
- before_script:
- - export NVM_DIR="$HOME/.nvm"
- - . "$NVM_DIR/nvm.sh"
script:
- yarn
- yarn test
-releases:test:
+.build_releases: &build_releases
stage: releases
+ allow_failure: false
image: duniter/release-builder:v1.0.1
tags:
- redshift-duniter-builder
- variables:
- DAY: $(date +%Y%m%d)
- HOUR: $(date +%H%M)
- SEC: $(date +%S)
+ when: manual
+ artifacts:
+ paths: &releases_artifacts
+ - work/bin/
+
+releases:test:
+ <<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
artifacts:
- paths:
- - work/bin/
- expire_in: 8h
- when: manual
+ paths: *releases_artifacts
+ expire_in: 4h
except:
- tags
releases:
- stage: releases
- image: duniter/release-builder:v1.0.1
- tags:
- - redshift-duniter-builder
+ <<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
artifacts:
- paths:
- - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb
- - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz
- - work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb
- expire_in: 8h
- when: manual
+ paths: *releases_artifacts
+ expire_in: 2 weeks
only:
- - tags
- - master
-
+ - tags
+
releases-message:
stage: releases-page
+ allow_failure: false
image: tensorflow/tensorflow:latest-py3
tags:
- redshift-duniter-builder
@@ -95,5 +89,4 @@ releases-message:
- python3 .gitlab/releaser.py
when: manual
only:
- - tags
- - master
+ - tags
--
GitLab
From ebe5909643eb43b76e424a89d66e4e834b291acb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Veyret?= <sveyret@axway.com>
Date: Tue, 23 Jan 2018 14:10:42 +0100
Subject: [PATCH 3/5] Rewrite Linux building, adding description
---
release/arch/linux/build-lin.sh | 159 ++++++++++++++++++++------------
1 file changed, 101 insertions(+), 58 deletions(-)
diff --git a/release/arch/linux/build-lin.sh b/release/arch/linux/build-lin.sh
index 585ee0019..b06938b45 100644
--- a/release/arch/linux/build-lin.sh
+++ b/release/arch/linux/build-lin.sh
@@ -11,6 +11,105 @@ else
exit 1
fi
+# ---------
+# Functions
+# ---------
+
+# Copy nw.js compiled module released library to node libraries.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy() {
+ [[ -z ${1} ]] && exit 1
+ cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
+ lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
+}
+
+# Copy nw.js compiled module library to node libraries, prefixing with node_.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy_node() {
+ [[ -z ${1} ]] && exit 1
+ cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
+ lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
+}
+
+# Compile the module with nw.js.
+# -
+# Parameters:
+# 1. Module name.
+# 2. Action to be done to module after compilation, if needed.
+nw_compile() {
+ [[ -z ${1} ]] && exit 1
+ cd ${1} || exit 1
+ node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
+ node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
+ [[ -z ${2} ]] || ${2} ${1}
+ cd ..
+}
+
+# Create description.
+# -
+# Parameters:
+# 1. Initial file name.
+# 2. Building type (either “desktop” or “server”).
+# 3. Category (OS, distribution).
+create_desc() {
+ cat >"${1}".desc <<-EOF
+ {
+ "version": "${DUNITER_TAG}",
+ "job": "${CI_JOB_NAME}",
+ "type": "${2^}",
+ "category": "${3}",
+ "arch": "x64"
+ }
+ EOF
+}
+
+# Desktop specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_desktop() {
+ cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
+}
+
+# Server specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_server() {
+ mkdir -p "${1}/lib/systemd/system" || exit 1
+ cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
+}
+
+# Debian package building.
+# -
+# Parameters:
+# 1. Building type (either “desktop” or “server”).
+# 2. Debian package name.
+build_deb_pack() {
+ rm -rf "${RELEASES}/duniter-x64"
+ mkdir "${RELEASES}/duniter-x64" || exit 1
+ cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
+ build_extra_${1} "${RELEASES}/duniter-x64"
+ mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
+ chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
+ chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
+ sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+ cd "${RELEASES}/${1}_/"
+ zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
+
+ sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+ cd "${RELEASES}"
+ fakeroot dpkg-deb --build duniter-x64 || exit 1
+ mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
+ create_desc "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" "${1}" "Linux (Ubuntu/Debian)"
+}
+
# -----------
# Prepare
# -----------
@@ -43,7 +142,7 @@ RELEASES="${WORK}/releases"
BIN="${WORK}/bin"
mkdir -p "${DOWNLOADS}" "${RELEASES}" "${BIN}" || exit 1
-rm -rf "${BIN}/"*.{deb,tar.gz} # Clean up
+rm -rf "${BIN}/"*.{deb,tar.gz}{,.desc} # Clean up
# -----------
# Downloads
@@ -83,27 +182,6 @@ cp -r "${RELEASES}/duniter" "${RELEASES}/server_" || exit 1
# Build Desktop version against nw.js
# -------------------------------------
-nw_copy() {
- [[ -z ${1} ]] && exit 1
- cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
- lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
-}
-
-nw_copy_node() {
- [[ -z ${1} ]] && exit 1
- cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
- lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
-}
-
-nw_compile() {
- [[ -z ${1} ]] && exit 1
- cd ${1} || exit 1
- node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
- node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
- [[ -z ${2} ]] || ${2} ${1}
- cd ..
-}
-
echo "${NW_RELEASE}"
# FIX: bug of nw.js, we need to patch first.
@@ -162,46 +240,11 @@ cp -r "${DOWNLOADS}/node-${NVER}-linux-x64" "${RELEASES}/server_/node" || exit 1
cd "${RELEASES}/desktop_"
tar czf "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" * || exit 1
+create_desc "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" "Desktop" "Linux (generic)"
# -----------------------
# Build Debian packages
# -----------------------
-# Parameters
-# 1: Building directory.
-build_extra_desktop() {
- cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
-}
-
-# Parameters
-# 1: Building directory.
-build_extra_server() {
- mkdir -p "${1}/lib/systemd/system" || exit 1
- cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
-}
-
-# Parameters
-# 1: either "server" or "desktop".
-# 2: package name for Debian.
-build_deb_pack() {
- rm -rf "${RELEASES}/duniter-x64"
- mkdir "${RELEASES}/duniter-x64" || exit 1
- cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
- build_extra_${1} "${RELEASES}/duniter-x64"
- mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
- chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
- chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
- sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
- cd "${RELEASES}/${1}_/"
- zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
-
- sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
- cd "${RELEASES}"
- fakeroot dpkg-deb --build duniter-x64 || exit 1
- mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
-}
-
build_deb_pack desktop duniter-desktop
build_deb_pack server duniter
--
GitLab
From 742481d4dcd075aae78976704dc5860f65feecdf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Veyret?= <sveyret@axway.com>
Date: Tue, 23 Jan 2018 17:36:06 +0100
Subject: [PATCH 4/5] Update release delivery
---
.gitlab-ci.yml | 33 +-
.gitlab/release_template.md | 13 +-
.gitlab/releaser.py | 701 +++++++++++++++++++++++++++++-------
3 files changed, 605 insertions(+), 142 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 19c14499e..b0bdb8346 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,8 +2,9 @@ stages:
- github-sync
- build
- test
- - releases
- - releases-page
+ - package
+ - prerelease
+ - release
push_to_github:
stage: github-sync
@@ -46,7 +47,7 @@ test:
- yarn test
.build_releases: &build_releases
- stage: releases
+ stage: package
allow_failure: false
image: duniter/release-builder:v1.0.1
tags:
@@ -66,7 +67,7 @@ releases:test:
except:
- tags
-releases:
+releases:x64:
<<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
@@ -76,17 +77,27 @@ releases:
only:
- tags
-releases-message:
- stage: releases-page
- allow_failure: false
+.release_jobs: &release_jobs
image: tensorflow/tensorflow:latest-py3
tags:
- redshift-duniter-builder
- variables:
- JOB_ARTIFACTS: 'releases'
- EXPECTED_ARTIFACTS: '["work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb","work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz","work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb"]'
script:
- python3 .gitlab/releaser.py
- when: manual
only:
- tags
+
+prerelease:
+ <<: *release_jobs
+ stage: prerelease
+ variables:
+ RELEASE_BIN_DIR: work/bin/
+ SOURCE_EXT: '["tar.gz", "zip"]'
+ RELEASE_JOB: publish
+
+publish:
+ <<: *release_jobs
+ stage: release
+ variables:
+ WIKI_RELEASE: Releases
+ allow_failure: false
+ when: manual
diff --git a/.gitlab/release_template.md b/.gitlab/release_template.md
index 99b23ff46..b9206fa0e 100644
--- a/.gitlab/release_template.md
+++ b/.gitlab/release_template.md
@@ -1,9 +1,12 @@
+<placeholder content="end-title" />
+<placeholder content="note">
{{current_message}}
+</placeholder>
-# Downloads
+## Downloads
+
+| Category | Arch | Type | Size | File |
+|----------|------|------|------|------|
{% for artifact in artifacts %}
-***
-[{{artifact.icon}} {{artifact.name}}]({{artifact.url}})
-_{{artifact.size}}_
-***
+| {{artifact.category}} | {{artifact.arch}} | {{artifact.type}} | {{artifact.size}} | [{{artifact.icon}} {{artifact.name}}]({{artifact.url}}) |
{% endfor %}
diff --git a/.gitlab/releaser.py b/.gitlab/releaser.py
index 833027fd0..21b522b9e 100644
--- a/.gitlab/releaser.py
+++ b/.gitlab/releaser.py
@@ -1,143 +1,592 @@
#!/usr/bin/python3
'''
-This module is meant to overload the release note in gitlab for the current project.
+This module is meant add release notes in gitlab for the current project.
Expects to find in environment following variables:
- CI_PROJECT_URL - Automatically set by gitlab-ci
- - CI_COMMIT_TAG - Automatically set by gitlab-ci
- CI_PROJECT_ID - Automatically set by gitlab-ci
- CI_COMMIT_TAG - Automatically set by gitlab-ci
+ - CI_PIPELINE_ID - Automatically set by gitlab-ci
+ - RELEASE_BIN_DIR - Directory where releases are to be found
+ - SOURCE_EXT - Source extensions
+ - RELEASE_JOB - Name of the release job
+ - WIKI_RELEASE - Wiki page where releases are stored
- RELEASER_TOKEN - Token used by technical user
- - JOB_ARTIFACTS - String containing job name containing all artifacts, to set manually
- - EXPECTED_ARTIFACTS - List containing all artifacts generated to set manually
'''
+import glob
+import jinja2
+import json
import math
+import os
import urllib.request
import urllib.error
-import json
-import os
-import jinja2
-def convert_size(size_bytes):
- '''Print proper size'''
- if size_bytes == 0:
- return '0B'
- size_name = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
- i = int(math.floor(math.log(size_bytes, 1024)))
- power = math.pow(1024, i)
- size = round(size_bytes / power, 2)
- return '%s %s' % (size, size_name[i])
-
-def get_current_message():
- '''Get current release message'''
- ci_project_id = os.environ['CI_PROJECT_ID']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- tag_url = 'https://git.duniter.org/api/v4/projects/'
- tag_url += ci_project_id
- tag_url += '/repository/tags/'
- tag_url += ci_commit_tag
- request = urllib.request.Request(tag_url)
- response = urllib.request.urlopen(request)
- response_data = response.read().decode()
- data = json.loads(response_data)
- if data['release'] is None:
- return False, ''
- else:
- return True, data['release']['description'].split('# Downloads')[0]
-
-def build_artifact_url(artifact, source):
- '''Given an artifact name, builds the url to download it'''
- job_artifacts = os.environ['JOB_ARTIFACTS']
- ci_project_url = os.environ['CI_PROJECT_URL']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- if source:
- source_url = ci_project_url
- source_url += '/repository/'
- source_url += ci_commit_tag
- source_url += '/archive.'
- source_url += artifact
- return source_url
- else:
- artifact_url = ci_project_url
- artifact_url += '/-/jobs/artifacts/'
- artifact_url += ci_commit_tag
- artifact_url += '/raw/'
- artifact_url += artifact
- artifact_url += '?job='
- artifact_url += job_artifacts
- return artifact_url
-
-def get_artifact_weight(location):
- '''Retrieve size of artifacts'''
- size = os.path.getsize(location)
- return convert_size(int(size))
-
-
-def build_compiled_message(current_message):
- '''Create a new release message using the release template'''
-
- expected_artifacts = os.environ['EXPECTED_ARTIFACTS']
- try:
- expected_artifacts = json.loads(expected_artifacts)
- except json.decoder.JSONDecodeError:
- print('CRITICAL EXPECTED_ARTIFACTS environment variable JSON probably malformed')
- print('CRITICAL Correct : \'["test_linux.txt","test_windows.txt"]\' ')
- print('CRITICAL Not Correct: "[\'test_linux.txt\',\'test_windows.txt\']" ')
+class FSItemSize:
+ '''
+ The size of a file system item.
+ '''
+
+ def __init__(self, bsize = None):
+ '''
+ :param bsize: Size of item in bytes.
+ :type bsize: int
+ '''
+ self.bsize = bsize
+
+ def __str__(self):
+ '''
+ :return: Human readable size.
+ :rtype: str
+ '''
+ if self.bsize is None:
+ return '(unknown)'
+ elif self.bsize == 0:
+ return '0 B'
+ size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
+ i = int(math.floor(math.log(self.bsize, 1024)))
+ power = math.pow(1024, i)
+ size = round(self.bsize / power, 2)
+ return '{} {}'.format(size, size_name[i])
+
+class Artifact:
+ '''
+ An artifact to be uploaded.
+ '''
+
+ def __init__(self, file_name, category, arch, dtype, icon):
+ '''
+ :param file_name: The name of the artifact file (may have directory).
+ :param category: The category (OS, distrib) for the artifact.
+ :param arch: The architecture name.
+ :param dtype: The delivery type (either server or desktop).
+ :param icon: The name of the icon to be used for artifact representation.
+ :type file_name: str
+ :type category: str
+ :type arch: str
+ :type dtype: str
+ :type icon: str
+ '''
+ self.file_name = file_name
+ self.category = category
+ self.arch = arch
+ self.dtype = dtype
+ self.icon = icon
+
+ def __lt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category < other.category or \
+ (self.category == other.category and self.arch < other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
+
+ def __le__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category <= other.category or \
+ (self.category == other.category and self.arch <= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
+
+ def __eq__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
+
+ def __ne__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
+
+ def __gt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category > other.category or \
+ (self.category == other.category and self.arch > other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
+
+ def __ge__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category >= other.category or \
+ (self.category == other.category and self.arch >= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
+
+ def to_dict(self):
+ '''
+ :return: A dictionnary containing artifact data.
+ :rtype: dict
+ '''
+ return {
+ 'name': self.file_name.split('/')[-1],
+ 'category': self.category,
+ 'arch': self.arch,
+ 'type': self.dtype,
+ 'url': self._build_url(),
+ 'size': self._get_size(),
+ 'icon': ':{}:'.format(self.icon)
+ }
+
+ def _get_size(self):
+ '''
+ :return: The size of the artifact.
+ :rtype: FSItemSize
+ '''
+ raise NotImplementedError()
+
+ def _build_url(self):
+ '''
+ :return: The URL which can be used to get this artifact.
+ :rtype: str
+ '''
+ raise NotImplementedError()
+
+class BinArtifact(Artifact):
+ '''
+ A binary artifact.
+ '''
+
+ def __init__(self, folder, desc_file, desc_ext):
+ '''
+ :param folder: The folder where files can be found.
+ :param desc_file: The name of the description file.
+ :param desc_ext: The extention of the description file.
+ :type folder: str
+ :type desc_file: str
+ :type desc_ext: str
+ '''
+ try:
+ description = json.load(open(desc_file))
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL Description file {} could not be read'.format(desc_file))
+ exit(1)
+
+ self.tag = description['version']
+ self.job = description['job']
+ file_name = desc_file[:-len(desc_ext)]
+ Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
+
+ def _get_size(self):
+ return FSItemSize(int(os.path.getsize(self.file_name)))
+
+ def _build_url(self):
+ return '{}/-/jobs/artifacts/{}/raw/{}?job={}'.format(
+ os.environ['CI_PROJECT_URL'], self.tag, self.file_name, self.job)
+
+class SourceArtifact(Artifact):
+ '''
+ A source artifact.
+ '''
+
+ def __init__(self, extention):
+ '''
+ :param extention: The extention of the source archive.
+ :type extention: str
+ '''
+ Artifact.__init__(self, 'archive.{}'.format(extention), 'Source code ({})'.format(extention), '', '', 'compression')
+
+ def _get_size(self):
+ return FSItemSize()
+
+ def _build_url(self):
+ return '{}/repository/{}/{}'.format(
+ os.environ['CI_PROJECT_URL'], os.environ['CI_COMMIT_TAG'], self.file_name)
+
+class PlaceHolder:
+ '''
+ Placeholder tags in Markdown texts.
+ '''
+ __PLACEHOLDER_PART = '<placeholder'
+ __PLACEHOLDER_START = '<placeholder content="{}">'
+ __PLACEHOLDER_STOP = '</placeholder>'
+ __PLACEHOLDER_FULL = '<placeholder content="{}" />'
+
+ def __init__(self, content_id):
+ '''
+ :param content_id: The identifier to be used for placeholder content.
+ :type content_id: str
+ '''
+ self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
+ self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
+ self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
+
+ def get_content(self, text):
+ '''
+ :param text: The text in which to extract content.
+ :type text: str
+ :return: The content between placeholder markers.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ text = text[pos + len(self.ph_start):]
+ pos = text.find(self.ph_stop)
+ if pos >= 0: text = text[:pos]
+ return text
+
+ def get_before(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content before (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if keep_mark: pos += len(self.ph_full)
+ text = text[:pos]
+ return text
+
+ def get_after(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content after (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if not keep_mark: pos += len(self.ph_full)
+ text = text[pos:]
+ return text
+
+ def replace_content(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been replaced.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ pos += len(self.ph_start)
+ text_before = text[:pos]
+ else:
+ pos = 0
+ text_before = ''
+ pos = text.find(self.ph_stop, pos)
+ if pos >= 0:
+ text_after = text[pos:]
+ else:
+ text_after = ''
+ return text_before + content + text_after
+
+ def insert_after(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been inserted.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0: pos += len(self.ph_full)
+ else: pos = 0
+ text_before = text[:pos]
+ text_after = text[pos:]
+ return text_before + content + text_after
+
+ def clear_all(text):
+ '''
+ Clear all placeholders from given text.
+ :param text: The text to clear.
+ :type text: str
+ :return: The clean text.
+ :rtype: str
+ '''
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
+ if pos < 0: break
+ end = text.find('>')
+ if end < 0: end = len(text)
+ text = text[:pos] + text[end + 1:]
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
+ if pos < 0: break
+ text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
+ return text
+
+class ProjectApi:
+ '''
+ Gitlab API project access.
+ '''
+ __PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
+
+ def __init__(self, url=''):
+ '''
+ :param url: The URL portion to add to base project URL (if needed).
+ :type url: str
+ '''
+ self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
+ self.base_url += url
+ self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
+
+ def build_request(self, url='', **params):
+ '''
+ Create the request to send to project API.
+ :param url: The portion of URL to add to base URL (if needed).
+ :param params: The optional parameters.
+ :type url: str
+ :type params: dict
+ :return: The request, ready to be used.
+ :rtype: urllib.request.Request
+ '''
+ request = urllib.request.Request(self.base_url + url, **params)
+ request.add_header(*self.token)
+ return request
+
+class Pipeline(ProjectApi):
+ '''
+ Pipeline data API.
+ '''
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/pipelines/{}'.format(os.environ['CI_PIPELINE_ID']))
+
+ def find_job_id(self, job_name):
+ '''
+ Find the id corresponding to given job name in the pipeline.
+ :param job_name: The job name.
+ :type job_name: str
+ :return: The identifier.
+ :rtype: int
+ '''
+ request = self.build_request('/jobs')
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ for job in json.loads(response_data):
+ if job['name'] == job_name: return job['id']
+ print('CRITICAL No job with given name {} found'.format(job_name))
exit(1)
- artifacts_list = []
- for artifact in expected_artifacts:
- artifact_dict = {
- 'name': artifact.split('/')[-1],
- 'url': build_artifact_url(artifact, False),
- 'size': get_artifact_weight(artifact),
- 'icon': ':package:'
+
+class Job(ProjectApi):
+ '''
+ Job data API.
+ '''
+
+ def __init__(self, job_id):
+ '''
+ :param job_id: The job id.
+ :type job_id: int
+ '''
+ ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
+
+ def keep_artifacts(self):
+ '''
+ Force artifacts to be kept forever.
+ '''
+ request = self.build_request('/artifacts/keep', method='POST')
+ urllib.request.urlopen(request)
+
+class ReleaseNote(ProjectApi):
+ '''
+ Release note API.
+ '''
+ __PH_TITLE = PlaceHolder('end-title')
+ __PH_NOTE = PlaceHolder('note')
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
+ self.message_read = False
+
+ def get_note(self):
+ '''
+ Get full release note.
+ :return: The note if it exists, None otherwise.
+ :rtype: str or None
+ '''
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ if data['release'] is None:
+ return None
+ else:
+ self.message_read = True
+ return data['release']['description']
+
+ def get_message(self):
+ '''
+ Get release message. Message is extracted from full note.
+ :return: The message if it exists, empty string otherwise.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ return ''
+ else:
+ return ReleaseNote.__PH_NOTE.get_content(data)
+
+ def get_note_body(self):
+ '''
+ Get release note body (without title). Body is extracted from full note.
+ :return: The body.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ print('CRITICAL No release information to publish')
+ exit(1)
+ return ReleaseNote.__PH_TITLE.get_after(data, True)
+
+ def send_note(self, note):
+ '''
+ Send the full release note. The current message should have been read
+ unless you are sure there are none.
+ :param note: The full note to send.
+ :type note: str
+ '''
+ method = 'PUT' if self.message_read else 'POST'
+ send_data = {
+ 'tag_name': os.environ['CI_COMMIT_TAG'],
+ 'description': note
}
- artifacts_list.append(artifact_dict)
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request('/release', data=send_data_serialized, method=method)
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
- j2_env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(
- os.path.dirname(os.path.abspath(__file__))
- ),
- trim_blocks=True
- )
- # pylint: disable=maybe-no-member
- template = j2_env.get_template('release_template.md')
- return template.render(
- current_message=current_message,
- artifacts=artifacts_list
- )
-
-
-def send_compiled_message(exists_release, compiled_message):
- '''Send to gitlab new message'''
- releaser_token = os.environ['RELEASER_TOKEN']
- ci_project_id = os.environ['CI_PROJECT_ID']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- release_url = 'https://git.duniter.org/api/v4/projects/'
- release_url += ci_project_id
- release_url += '/repository/tags/'
- release_url += ci_commit_tag
- release_url += '/release'
- if exists_release:
- # We need to send a PUT request
- method = 'PUT'
- else:
- # We need to send a POST request
- method = 'POST'
- send_data = {
- 'tag_name':ci_commit_tag,
- 'description':compiled_message
+class ReleaseWikiPage(ProjectApi):
+ '''
+ Release Wiki page API.
+ '''
+ __PH_TAG = PlaceHolder('tag')
+ __PH_NOTE = PlaceHolder('note')
+ __PH_PREVIOUS = PlaceHolder('previous-beg')
+ __PREVIOUS_NOTE = '\n\n## {}\n\n{}'
+
+ def __init__(self):
+ if not 'WIKI_RELEASE' in os.environ:
+ print('CRITICAL WIKI_RELEASE variable is not defined')
+ exit(1)
+ ProjectApi.__init__(self, '/wikis/{}'.format(os.environ['WIKI_RELEASE']))
+
+ # Query existing page
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ self.page_content = data['content']
+
+ def add_release(self, tag, note):
+ '''
+ Add the release to the Wiki page.
+ '''
+ prev_tag = ReleaseWikiPage.__PH_TAG.get_content(self.page_content)
+ prev_note = ReleaseWikiPage.__PH_NOTE.get_content(self.page_content)
+ self.page_content = ReleaseWikiPage.__PH_TAG.replace_content(self.page_content, tag)
+ self.page_content = ReleaseWikiPage.__PH_NOTE.replace_content(self.page_content, note)
+ self.page_content = ReleaseWikiPage.__PH_PREVIOUS.insert_after(
+ self.page_content,
+ ReleaseWikiPage.__PREVIOUS_NOTE.format(prev_tag, prev_note))
+
+ def save(self):
+ send_data = {
+ 'content': self.page_content,
+ 'format': 'markdown',
+ 'slug': os.environ['WIKI_RELEASE'],
+ 'title': os.environ['WIKI_RELEASE']
}
- send_data_serialized = json.dumps(send_data).encode('utf-8')
- request = urllib.request.Request(release_url, data=send_data_serialized, method=method)
- request.add_header('Private-Token', releaser_token)
- request.add_header('Content-Type', 'application/json')
- response = urllib.request.urlopen(request)
-
-def main():
- '''Execute main scenario'''
- exists_release, current_message = get_current_message()
- compiled_message = build_compiled_message(current_message)
- send_compiled_message(exists_release, compiled_message)
- print('Artifacts uploaded successfully')
-main()
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request(data=send_data_serialized, method='PUT')
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
+
+class Releaser:
+ '''
+ The main releaser class
+ '''
+ __PRERELEASE = '# :gift: Pre-release\n\n[Go to Pipeline page :arrow_forward:](https://git.duniter.org/sveyret/duniter/pipelines/{})\n\n'
+ __RELEASE = '# :white_check_mark: Release\n\n'
+ __DESC_EXT = '.desc'
+
+ def __init__(self):
+ if 'RELEASE_BIN_DIR' in os.environ:
+ self.release_bin_dir = os.environ['RELEASE_BIN_DIR']
+ if not self.release_bin_dir.endswith('/'): self.release_bin_dir += '/'
+ else: self.release_bin_dir = None
+ if 'SOURCE_EXT' in os.environ:
+ self.source_ext = os.environ['SOURCE_EXT']
+ try:
+ self.source_ext = json.loads(self.source_ext)
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL SOURCE_EXT environment variable JSON probably malformed')
+ print('CRITICAL Correct : \'["zip","tar.gz"]\' ')
+ print('CRITICAL Not Correct: "[\'zip\',\'tar.gz\']" ')
+ exit(1)
+ else: self.source_ext = None
+ if 'RELEASE_JOB' in os.environ:
+ self.release_job = os.environ['RELEASE_JOB']
+ else: self.release_job = None
+
+ def release(self):
+ if self.release_bin_dir is None or self.source_ext is None or self.release_job is None:
+ self.publish_release()
+ else:
+ self.publish_prerelease()
+
+ def publish_prerelease(self):
+ '''
+ Main job to publish a pre-release.
+ '''
+ releaseNote = ReleaseNote()
+ current_message = releaseNote.get_message()
+ artifacts_list = []
+
+ # Binary releases
+ artifacts_list += list(filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'],
+ map(lambda d: BinArtifact(self.release_bin_dir, d, Releaser.__DESC_EXT),
+ glob.glob('{}*{}'.format(self.release_bin_dir, Releaser.__DESC_EXT)))))
+ artifacts_list.sort()
+
+ # Sources
+ artifacts_list += list(map(lambda e: SourceArtifact(e), self.source_ext))
+
+ # Load template
+ j2_env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(
+ os.path.dirname(os.path.abspath(__file__))
+ ),
+ trim_blocks=True
+ )
+ # pylint: disable=maybe-no-member
+ template = j2_env.get_template('release_template.md')
+
+ # Send result
+ note = template.render(
+ current_message = current_message,
+ artifacts = list(map(lambda a: a.to_dict(), artifacts_list))
+ )
+ title_line = Releaser.__PRERELEASE.format(os.environ['CI_PIPELINE_ID'])
+ releaseNote.send_note(title_line + note)
+
+ print('Pre-release published')
+
+ def publish_release(self):
+ '''
+ Main job to publish the final release.
+ '''
+ # Change release note
+ releaseNote = ReleaseNote()
+ note = releaseNote.get_note_body()
+ title_line = Releaser.__RELEASE
+ releaseNote.send_note(title_line + note)
+
+ # Update Wiki release page
+ wiki_page = ReleaseWikiPage()
+ wiki_page.add_release(os.environ['CI_COMMIT_TAG'], PlaceHolder.clear_all(note))
+ wiki_page.save()
+
+ # Keep artifacts
+ artifacts_list = list(filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'],
+ map(lambda d: BinArtifact(self.release_bin_dir, d, Releaser.__DESC_EXT),
+ glob.glob('{}*{}'.format(self.release_bin_dir, Releaser.__DESC_EXT)))))
+ jobs = []
+ for artifact in artifacts_list:
+ if not artifact.job in jobs:
+ jobs.append(artifact.job)
+ jobs = map(lambda j: Pipeline.find_job_id(j), jobs)
+ for job_id in jobs: Job(job_id).keep_artifacts()
+
+ print('Release published')
+
+Releaser().release()
--
GitLab
From 99b7110f5cb633d49cb995de87df7f0742d88fde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Veyret?= <stephane.veyret@neptura.org>
Date: Fri, 26 Jan 2018 08:04:41 +0100
Subject: [PATCH 5/5] Convert releaser into module
---
.gitignore | 3 +
.gitlab-ci.yml | 4 +-
.gitlab/release_template.md | 22 ++
.gitlab/releaser.py | 592 ----------------------------
.gitlab/releaser/__init__.py | 12 +
.gitlab/releaser/__main__.py | 3 +
.gitlab/releaser/artifact.py | 84 ++++
.gitlab/releaser/binartifact.py | 37 ++
.gitlab/releaser/fsitemsize.py | 28 ++
.gitlab/releaser/job.py | 22 ++
.gitlab/releaser/pipeline.py | 29 ++
.gitlab/releaser/placeholder.py | 120 ++++++
.gitlab/releaser/projectapi.py | 31 ++
.gitlab/releaser/releasenote.py | 74 ++++
.gitlab/releaser/releaser.py | 109 +++++
.gitlab/releaser/releasewikipage.py | 59 +++
.gitlab/releaser/sourceartifact.py | 23 ++
.gitlab/releaser/template.py | 31 ++
18 files changed, 689 insertions(+), 594 deletions(-)
delete mode 100644 .gitlab/releaser.py
create mode 100644 .gitlab/releaser/__init__.py
create mode 100644 .gitlab/releaser/__main__.py
create mode 100644 .gitlab/releaser/artifact.py
create mode 100644 .gitlab/releaser/binartifact.py
create mode 100644 .gitlab/releaser/fsitemsize.py
create mode 100644 .gitlab/releaser/job.py
create mode 100644 .gitlab/releaser/pipeline.py
create mode 100644 .gitlab/releaser/placeholder.py
create mode 100644 .gitlab/releaser/projectapi.py
create mode 100644 .gitlab/releaser/releasenote.py
create mode 100644 .gitlab/releaser/releaser.py
create mode 100644 .gitlab/releaser/releasewikipage.py
create mode 100644 .gitlab/releaser/sourceartifact.py
create mode 100644 .gitlab/releaser/template.py
diff --git a/.gitignore b/.gitignore
index dcf30dda1..0a79bb0ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,9 @@ gui/nw
vagrant/*.log
vagrant/duniter
+# Python compiled
+*.pyc
+
# Releases
/work
*.deb
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b0bdb8346..d75bd05da 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -82,7 +82,7 @@ releases:x64:
tags:
- redshift-duniter-builder
script:
- - python3 .gitlab/releaser.py
+ - python3 .gitlab/releaser
only:
- tags
@@ -92,12 +92,12 @@ prerelease:
variables:
RELEASE_BIN_DIR: work/bin/
SOURCE_EXT: '["tar.gz", "zip"]'
- RELEASE_JOB: publish
publish:
<<: *release_jobs
stage: release
variables:
+ RELEASE_BIN_DIR: work/bin/
WIKI_RELEASE: Releases
allow_failure: false
when: manual
diff --git a/.gitlab/release_template.md b/.gitlab/release_template.md
index b9206fa0e..1e04757fc 100644
--- a/.gitlab/release_template.md
+++ b/.gitlab/release_template.md
@@ -1,3 +1,16 @@
+{% block prerelease %}
+# :gift: Pre-release
+
+[Go to Pipeline page :arrow_forward:](https://git.duniter.org/sveyret/duniter/pipelines/{{pipeline}})
+
+{% endblock %}
+
+{% block release %}
+# :white_check_mark: Release
+
+{% endblock %}
+
+{% block notebody %}
<placeholder content="end-title" />
<placeholder content="note">
{{current_message}}
@@ -10,3 +23,12 @@
{% for artifact in artifacts %}
| {{artifact.category}} | {{artifact.arch}} | {{artifact.type}} | {{artifact.size}} | [{{artifact.icon}} {{artifact.name}}]({{artifact.url}}) |
{% endfor %}
+{% endblock %}
+
+{% block previouswiki %}
+
+
+## {{tag}}
+
+{{body}}
+{% endblock %}
diff --git a/.gitlab/releaser.py b/.gitlab/releaser.py
deleted file mode 100644
index 21b522b9e..000000000
--- a/.gitlab/releaser.py
+++ /dev/null
@@ -1,592 +0,0 @@
-#!/usr/bin/python3
-'''
-This module is meant add release notes in gitlab for the current project.
-Expects to find in environment following variables:
- - CI_PROJECT_URL - Automatically set by gitlab-ci
- - CI_PROJECT_ID - Automatically set by gitlab-ci
- - CI_COMMIT_TAG - Automatically set by gitlab-ci
- - CI_PIPELINE_ID - Automatically set by gitlab-ci
- - RELEASE_BIN_DIR - Directory where releases are to be found
- - SOURCE_EXT - Source extensions
- - RELEASE_JOB - Name of the release job
- - WIKI_RELEASE - Wiki page where releases are stored
- - RELEASER_TOKEN - Token used by technical user
-'''
-
-import glob
-import jinja2
-import json
-import math
-import os
-import urllib.request
-import urllib.error
-
-class FSItemSize:
- '''
- The size of a file system item.
- '''
-
- def __init__(self, bsize = None):
- '''
- :param bsize: Size of item in bytes.
- :type bsize: int
- '''
- self.bsize = bsize
-
- def __str__(self):
- '''
- :return: Human readable size.
- :rtype: str
- '''
- if self.bsize is None:
- return '(unknown)'
- elif self.bsize == 0:
- return '0 B'
- size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
- i = int(math.floor(math.log(self.bsize, 1024)))
- power = math.pow(1024, i)
- size = round(self.bsize / power, 2)
- return '{} {}'.format(size, size_name[i])
-
-class Artifact:
- '''
- An artifact to be uploaded.
- '''
-
- def __init__(self, file_name, category, arch, dtype, icon):
- '''
- :param file_name: The name of the artifact file (may have directory).
- :param category: The category (OS, distrib) for the artifact.
- :param arch: The architecture name.
- :param dtype: The delivery type (either server or desktop).
- :param icon: The name of the icon to be used for artifact representation.
- :type file_name: str
- :type category: str
- :type arch: str
- :type dtype: str
- :type icon: str
- '''
- self.file_name = file_name
- self.category = category
- self.arch = arch
- self.dtype = dtype
- self.icon = icon
-
- def __lt__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category < other.category or \
- (self.category == other.category and self.arch < other.arch) or \
- (self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
-
- def __le__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category <= other.category or \
- (self.category == other.category and self.arch <= other.arch) or \
- (self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
-
- def __eq__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
-
- def __ne__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
-
- def __gt__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category > other.category or \
- (self.category == other.category and self.arch > other.arch) or \
- (self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
-
- def __ge__(self, other):
- if not isinstance(other, Artifact): raise TypeError()
- return self.category >= other.category or \
- (self.category == other.category and self.arch >= other.arch) or \
- (self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
-
- def to_dict(self):
- '''
- :return: A dictionnary containing artifact data.
- :rtype: dict
- '''
- return {
- 'name': self.file_name.split('/')[-1],
- 'category': self.category,
- 'arch': self.arch,
- 'type': self.dtype,
- 'url': self._build_url(),
- 'size': self._get_size(),
- 'icon': ':{}:'.format(self.icon)
- }
-
- def _get_size(self):
- '''
- :return: The size of the artifact.
- :rtype: FSItemSize
- '''
- raise NotImplementedError()
-
- def _build_url(self):
- '''
- :return: The URL which can be used to get this artifact.
- :rtype: str
- '''
- raise NotImplementedError()
-
-class BinArtifact(Artifact):
- '''
- A binary artifact.
- '''
-
- def __init__(self, folder, desc_file, desc_ext):
- '''
- :param folder: The folder where files can be found.
- :param desc_file: The name of the description file.
- :param desc_ext: The extention of the description file.
- :type folder: str
- :type desc_file: str
- :type desc_ext: str
- '''
- try:
- description = json.load(open(desc_file))
- except json.decoder.JSONDecodeError:
- print('CRITICAL Description file {} could not be read'.format(desc_file))
- exit(1)
-
- self.tag = description['version']
- self.job = description['job']
- file_name = desc_file[:-len(desc_ext)]
- Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
-
- def _get_size(self):
- return FSItemSize(int(os.path.getsize(self.file_name)))
-
- def _build_url(self):
- return '{}/-/jobs/artifacts/{}/raw/{}?job={}'.format(
- os.environ['CI_PROJECT_URL'], self.tag, self.file_name, self.job)
-
-class SourceArtifact(Artifact):
- '''
- A source artifact.
- '''
-
- def __init__(self, extention):
- '''
- :param extention: The extention of the source archive.
- :type extention: str
- '''
- Artifact.__init__(self, 'archive.{}'.format(extention), 'Source code ({})'.format(extention), '', '', 'compression')
-
- def _get_size(self):
- return FSItemSize()
-
- def _build_url(self):
- return '{}/repository/{}/{}'.format(
- os.environ['CI_PROJECT_URL'], os.environ['CI_COMMIT_TAG'], self.file_name)
-
-class PlaceHolder:
- '''
- Placeholder tags in Markdown texts.
- '''
- __PLACEHOLDER_PART = '<placeholder'
- __PLACEHOLDER_START = '<placeholder content="{}">'
- __PLACEHOLDER_STOP = '</placeholder>'
- __PLACEHOLDER_FULL = '<placeholder content="{}" />'
-
- def __init__(self, content_id):
- '''
- :param content_id: The identifier to be used for placeholder content.
- :type content_id: str
- '''
- self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
- self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
- self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
-
- def get_content(self, text):
- '''
- :param text: The text in which to extract content.
- :type text: str
- :return: The content between placeholder markers.
- :rtype: str
- '''
- pos = text.find(self.ph_start)
- if pos >= 0:
- text = text[pos + len(self.ph_start):]
- pos = text.find(self.ph_stop)
- if pos >= 0: text = text[:pos]
- return text
-
- def get_before(self, text, keep_mark=False):
- '''
- :param text: The text in which to extract content.
- :param keep_mark: If true, the mark is kept in final text.
- :type text: str
- :type keep_mark: bool
- :return: The content before (full) placeholder marker.
- :rtype: str
- '''
- pos = text.find(self.ph_full)
- if pos >= 0:
- if keep_mark: pos += len(self.ph_full)
- text = text[:pos]
- return text
-
- def get_after(self, text, keep_mark=False):
- '''
- :param text: The text in which to extract content.
- :param keep_mark: If true, the mark is kept in final text.
- :type text: str
- :type keep_mark: bool
- :return: The content after (full) placeholder marker.
- :rtype: str
- '''
- pos = text.find(self.ph_full)
- if pos >= 0:
- if not keep_mark: pos += len(self.ph_full)
- text = text[pos:]
- return text
-
- def replace_content(self, text, content):
- '''
- :param text: The text in which to extract content.
- :param content: The new content to insert.
- :type text: str
- :type content: str
- :return: The text where content has been replaced.
- :rtype: str
- '''
- pos = text.find(self.ph_start)
- if pos >= 0:
- pos += len(self.ph_start)
- text_before = text[:pos]
- else:
- pos = 0
- text_before = ''
- pos = text.find(self.ph_stop, pos)
- if pos >= 0:
- text_after = text[pos:]
- else:
- text_after = ''
- return text_before + content + text_after
-
- def insert_after(self, text, content):
- '''
- :param text: The text in which to extract content.
- :param content: The new content to insert.
- :type text: str
- :type content: str
- :return: The text where content has been inserted.
- :rtype: str
- '''
- pos = text.find(self.ph_full)
- if pos >= 0: pos += len(self.ph_full)
- else: pos = 0
- text_before = text[:pos]
- text_after = text[pos:]
- return text_before + content + text_after
-
- def clear_all(text):
- '''
- Clear all placeholders from given text.
- :param text: The text to clear.
- :type text: str
- :return: The clean text.
- :rtype: str
- '''
- while True:
- pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
- if pos < 0: break
- end = text.find('>')
- if end < 0: end = len(text)
- text = text[:pos] + text[end + 1:]
- while True:
- pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
- if pos < 0: break
- text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
- return text
-
-class ProjectApi:
- '''
- Gitlab API project access.
- '''
- __PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
-
- def __init__(self, url=''):
- '''
- :param url: The URL portion to add to base project URL (if needed).
- :type url: str
- '''
- self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
- self.base_url += url
- self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
-
- def build_request(self, url='', **params):
- '''
- Create the request to send to project API.
- :param url: The portion of URL to add to base URL (if needed).
- :param params: The optional parameters.
- :type url: str
- :type params: dict
- :return: The request, ready to be used.
- :rtype: urllib.request.Request
- '''
- request = urllib.request.Request(self.base_url + url, **params)
- request.add_header(*self.token)
- return request
-
-class Pipeline(ProjectApi):
- '''
- Pipeline data API.
- '''
-
- def __init__(self):
- ProjectApi.__init__(self, '/pipelines/{}'.format(os.environ['CI_PIPELINE_ID']))
-
- def find_job_id(self, job_name):
- '''
- Find the id corresponding to given job name in the pipeline.
- :param job_name: The job name.
- :type job_name: str
- :return: The identifier.
- :rtype: int
- '''
- request = self.build_request('/jobs')
- response = urllib.request.urlopen(request)
- response_data = response.read().decode()
- for job in json.loads(response_data):
- if job['name'] == job_name: return job['id']
- print('CRITICAL No job with given name {} found'.format(job_name))
- exit(1)
-
-class Job(ProjectApi):
- '''
- Job data API.
- '''
-
- def __init__(self, job_id):
- '''
- :param job_id: The job id.
- :type job_id: int
- '''
- ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
-
- def keep_artifacts(self):
- '''
- Force artifacts to be kept forever.
- '''
- request = self.build_request('/artifacts/keep', method='POST')
- urllib.request.urlopen(request)
-
-class ReleaseNote(ProjectApi):
- '''
- Release note API.
- '''
- __PH_TITLE = PlaceHolder('end-title')
- __PH_NOTE = PlaceHolder('note')
-
- def __init__(self):
- ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
- self.message_read = False
-
- def get_note(self):
- '''
- Get full release note.
- :return: The note if it exists, None otherwise.
- :rtype: str or None
- '''
- request = self.build_request()
- response = urllib.request.urlopen(request)
- response_data = response.read().decode()
- data = json.loads(response_data)
- if data['release'] is None:
- return None
- else:
- self.message_read = True
- return data['release']['description']
-
- def get_message(self):
- '''
- Get release message. Message is extracted from full note.
- :return: The message if it exists, empty string otherwise.
- :rtype: str
- '''
- data = self.get_note()
- if data is None:
- return ''
- else:
- return ReleaseNote.__PH_NOTE.get_content(data)
-
- def get_note_body(self):
- '''
- Get release note body (without title). Body is extracted from full note.
- :return: The body.
- :rtype: str
- '''
- data = self.get_note()
- if data is None:
- print('CRITICAL No release information to publish')
- exit(1)
- return ReleaseNote.__PH_TITLE.get_after(data, True)
-
- def send_note(self, note):
- '''
- Send the full release note. The current message should have been read
- unless you are sure there are none.
- :param note: The full note to send.
- :type note: str
- '''
- method = 'PUT' if self.message_read else 'POST'
- send_data = {
- 'tag_name': os.environ['CI_COMMIT_TAG'],
- 'description': note
- }
- send_data_serialized = json.dumps(send_data).encode('utf-8')
- request = self.build_request('/release', data=send_data_serialized, method=method)
- request.add_header('Content-Type', 'application/json')
- urllib.request.urlopen(request)
-
-class ReleaseWikiPage(ProjectApi):
- '''
- Release Wiki page API.
- '''
- __PH_TAG = PlaceHolder('tag')
- __PH_NOTE = PlaceHolder('note')
- __PH_PREVIOUS = PlaceHolder('previous-beg')
- __PREVIOUS_NOTE = '\n\n## {}\n\n{}'
-
- def __init__(self):
- if not 'WIKI_RELEASE' in os.environ:
- print('CRITICAL WIKI_RELEASE variable is not defined')
- exit(1)
- ProjectApi.__init__(self, '/wikis/{}'.format(os.environ['WIKI_RELEASE']))
-
- # Query existing page
- request = self.build_request()
- response = urllib.request.urlopen(request)
- response_data = response.read().decode()
- data = json.loads(response_data)
- self.page_content = data['content']
-
- def add_release(self, tag, note):
- '''
- Add the release to the Wiki page.
- '''
- prev_tag = ReleaseWikiPage.__PH_TAG.get_content(self.page_content)
- prev_note = ReleaseWikiPage.__PH_NOTE.get_content(self.page_content)
- self.page_content = ReleaseWikiPage.__PH_TAG.replace_content(self.page_content, tag)
- self.page_content = ReleaseWikiPage.__PH_NOTE.replace_content(self.page_content, note)
- self.page_content = ReleaseWikiPage.__PH_PREVIOUS.insert_after(
- self.page_content,
- ReleaseWikiPage.__PREVIOUS_NOTE.format(prev_tag, prev_note))
-
- def save(self):
- send_data = {
- 'content': self.page_content,
- 'format': 'markdown',
- 'slug': os.environ['WIKI_RELEASE'],
- 'title': os.environ['WIKI_RELEASE']
- }
- send_data_serialized = json.dumps(send_data).encode('utf-8')
- request = self.build_request(data=send_data_serialized, method='PUT')
- request.add_header('Content-Type', 'application/json')
- urllib.request.urlopen(request)
-
-class Releaser:
- '''
- The main releaser class
- '''
- __PRERELEASE = '# :gift: Pre-release\n\n[Go to Pipeline page :arrow_forward:](https://git.duniter.org/sveyret/duniter/pipelines/{})\n\n'
- __RELEASE = '# :white_check_mark: Release\n\n'
- __DESC_EXT = '.desc'
-
- def __init__(self):
- if 'RELEASE_BIN_DIR' in os.environ:
- self.release_bin_dir = os.environ['RELEASE_BIN_DIR']
- if not self.release_bin_dir.endswith('/'): self.release_bin_dir += '/'
- else: self.release_bin_dir = None
- if 'SOURCE_EXT' in os.environ:
- self.source_ext = os.environ['SOURCE_EXT']
- try:
- self.source_ext = json.loads(self.source_ext)
- except json.decoder.JSONDecodeError:
- print('CRITICAL SOURCE_EXT environment variable JSON probably malformed')
- print('CRITICAL Correct : \'["zip","tar.gz"]\' ')
- print('CRITICAL Not Correct: "[\'zip\',\'tar.gz\']" ')
- exit(1)
- else: self.source_ext = None
- if 'RELEASE_JOB' in os.environ:
- self.release_job = os.environ['RELEASE_JOB']
- else: self.release_job = None
-
- def release(self):
- if self.release_bin_dir is None or self.source_ext is None or self.release_job is None:
- self.publish_release()
- else:
- self.publish_prerelease()
-
- def publish_prerelease(self):
- '''
- Main job to publish a pre-release.
- '''
- releaseNote = ReleaseNote()
- current_message = releaseNote.get_message()
- artifacts_list = []
-
- # Binary releases
- artifacts_list += list(filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'],
- map(lambda d: BinArtifact(self.release_bin_dir, d, Releaser.__DESC_EXT),
- glob.glob('{}*{}'.format(self.release_bin_dir, Releaser.__DESC_EXT)))))
- artifacts_list.sort()
-
- # Sources
- artifacts_list += list(map(lambda e: SourceArtifact(e), self.source_ext))
-
- # Load template
- j2_env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(
- os.path.dirname(os.path.abspath(__file__))
- ),
- trim_blocks=True
- )
- # pylint: disable=maybe-no-member
- template = j2_env.get_template('release_template.md')
-
- # Send result
- note = template.render(
- current_message = current_message,
- artifacts = list(map(lambda a: a.to_dict(), artifacts_list))
- )
- title_line = Releaser.__PRERELEASE.format(os.environ['CI_PIPELINE_ID'])
- releaseNote.send_note(title_line + note)
-
- print('Pre-release published')
-
- def publish_release(self):
- '''
- Main job to publish the final release.
- '''
- # Change release note
- releaseNote = ReleaseNote()
- note = releaseNote.get_note_body()
- title_line = Releaser.__RELEASE
- releaseNote.send_note(title_line + note)
-
- # Update Wiki release page
- wiki_page = ReleaseWikiPage()
- wiki_page.add_release(os.environ['CI_COMMIT_TAG'], PlaceHolder.clear_all(note))
- wiki_page.save()
-
- # Keep artifacts
- artifacts_list = list(filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'],
- map(lambda d: BinArtifact(self.release_bin_dir, d, Releaser.__DESC_EXT),
- glob.glob('{}*{}'.format(self.release_bin_dir, Releaser.__DESC_EXT)))))
- jobs = []
- for artifact in artifacts_list:
- if not artifact.job in jobs:
- jobs.append(artifact.job)
- jobs = map(lambda j: Pipeline.find_job_id(j), jobs)
- for job_id in jobs: Job(job_id).keep_artifacts()
-
- print('Release published')
-
-Releaser().release()
diff --git a/.gitlab/releaser/__init__.py b/.gitlab/releaser/__init__.py
new file mode 100644
index 000000000..4f0a11e46
--- /dev/null
+++ b/.gitlab/releaser/__init__.py
@@ -0,0 +1,12 @@
+'''
+This module is meant add release notes in gitlab for the current project.
+Expects to find in environment following variables:
+ - CI_PROJECT_URL - Automatically set by gitlab-ci
+ - CI_PROJECT_ID - Automatically set by gitlab-ci
+ - CI_COMMIT_TAG - Automatically set by gitlab-ci
+ - CI_PIPELINE_ID - Automatically set by gitlab-ci
+ - RELEASE_BIN_DIR - Directory where releases are to be found
+ - SOURCE_EXT - Source extensions (pre-release only)
+ - WIKI_RELEASE - Wiki page where releases are stored (release only)
+ - RELEASER_TOKEN - Token used by technical user
+'''
diff --git a/.gitlab/releaser/__main__.py b/.gitlab/releaser/__main__.py
new file mode 100644
index 000000000..83a061b83
--- /dev/null
+++ b/.gitlab/releaser/__main__.py
@@ -0,0 +1,3 @@
+from releaser import Releaser
+
+Releaser().release()
diff --git a/.gitlab/releaser/artifact.py b/.gitlab/releaser/artifact.py
new file mode 100644
index 000000000..e0d37f33a
--- /dev/null
+++ b/.gitlab/releaser/artifact.py
@@ -0,0 +1,84 @@
+class Artifact:
+ '''
+ An artifact to be uploaded.
+ '''
+
+ def __init__(self, file_name, category, arch, dtype, icon):
+ '''
+ :param file_name: The name of the artifact file (may have directory).
+ :param category: The category (OS, distrib) for the artifact.
+ :param arch: The architecture name.
+ :param dtype: The delivery type (either server or desktop).
+ :param icon: The name of the icon to be used for artifact representation.
+ :type file_name: str
+ :type category: str
+ :type arch: str
+ :type dtype: str
+ :type icon: str
+ '''
+ self.file_name = file_name
+ self.category = category
+ self.arch = arch
+ self.dtype = dtype
+ self.icon = icon
+
+ def __lt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category < other.category or \
+ (self.category == other.category and self.arch < other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
+
+ def __le__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category <= other.category or \
+ (self.category == other.category and self.arch <= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
+
+ def __eq__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
+
+ def __ne__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
+
+ def __gt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category > other.category or \
+ (self.category == other.category and self.arch > other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
+
+ def __ge__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category >= other.category or \
+ (self.category == other.category and self.arch >= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
+
+ def to_dict(self):
+ '''
+ :return: A dictionnary containing artifact data.
+ :rtype: dict
+ '''
+ return {
+ 'name': self.file_name.split('/')[-1],
+ 'category': self.category,
+ 'arch': self.arch,
+ 'type': self.dtype,
+ 'url': self._build_url(),
+ 'size': self._get_size(),
+ 'icon': ':{}:'.format(self.icon)
+ }
+
+ def _get_size(self):
+ '''
+ :return: The size of the artifact.
+ :rtype: FSItemSize
+ '''
+ raise NotImplementedError()
+
+ def _build_url(self):
+ '''
+ :return: The URL which can be used to get this artifact.
+ :rtype: str
+ '''
+ raise NotImplementedError()
diff --git a/.gitlab/releaser/binartifact.py b/.gitlab/releaser/binartifact.py
new file mode 100644
index 000000000..45c617a6c
--- /dev/null
+++ b/.gitlab/releaser/binartifact.py
@@ -0,0 +1,37 @@
+import json
+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class BinArtifact(Artifact):
+ '''
+ A binary artifact.
+ '''
+
+ def __init__(self, folder, desc_file, desc_ext):
+ '''
+ :param folder: The folder where files can be found.
+ :param desc_file: The name of the description file.
+ :param desc_ext: The extention of the description file.
+ :type folder: str
+ :type desc_file: str
+ :type desc_ext: str
+ '''
+ try:
+ description = json.load(open(desc_file))
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL Description file {} could not be read'.format(desc_file))
+ exit(1)
+
+ self.tag = description['version']
+ self.job = description['job']
+ file_name = desc_file[:-len(desc_ext)]
+ Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
+
+ def _get_size(self):
+ return FSItemSize(int(os.path.getsize(self.file_name)))
+
+ def _build_url(self):
+ return '{}/-/jobs/artifacts/{}/raw/{}?job={}'.format(
+ os.environ['CI_PROJECT_URL'], self.tag, self.file_name, self.job)
diff --git a/.gitlab/releaser/fsitemsize.py b/.gitlab/releaser/fsitemsize.py
new file mode 100644
index 000000000..a6c8be232
--- /dev/null
+++ b/.gitlab/releaser/fsitemsize.py
@@ -0,0 +1,28 @@
+import math
+
+class FSItemSize:
+ '''
+ The size of a file system item.
+ '''
+
+ def __init__(self, bsize = None):
+ '''
+ :param bsize: Size of item in bytes.
+ :type bsize: int
+ '''
+ self.bsize = bsize
+
+ def __str__(self):
+ '''
+ :return: Human readable size.
+ :rtype: str
+ '''
+ if self.bsize is None:
+ return '(unknown)'
+ elif self.bsize == 0:
+ return '0 B'
+ size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
+ i = int(math.floor(math.log(self.bsize, 1024)))
+ power = math.pow(1024, i)
+ size = round(self.bsize / power, 2)
+ return '{} {}'.format(size, size_name[i])
diff --git a/.gitlab/releaser/job.py b/.gitlab/releaser/job.py
new file mode 100644
index 000000000..2dc55c08f
--- /dev/null
+++ b/.gitlab/releaser/job.py
@@ -0,0 +1,22 @@
+import urllib.request
+
+from projectapi import ProjectApi
+
+class Job(ProjectApi):
+ '''
+ Job data API.
+ '''
+
+ def __init__(self, job_id):
+ '''
+ :param job_id: The job id.
+ :type job_id: int
+ '''
+ ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
+
+ def keep_artifacts(self):
+ '''
+ Force artifacts to be kept forever.
+ '''
+ request = self.build_request('/artifacts/keep', method='POST')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/pipeline.py b/.gitlab/releaser/pipeline.py
new file mode 100644
index 000000000..48e72216e
--- /dev/null
+++ b/.gitlab/releaser/pipeline.py
@@ -0,0 +1,29 @@
+import json
+import os
+import urllib.request
+
+from projectapi import ProjectApi
+
+class Pipeline(ProjectApi):
+ '''
+ Pipeline data API.
+ '''
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/pipelines/{}'.format(os.environ['CI_PIPELINE_ID']))
+
+ def find_job_id(self, job_name):
+ '''
+ Find the id corresponding to given job name in the pipeline.
+ :param job_name: The job name.
+ :type job_name: str
+ :return: The identifier.
+ :rtype: int
+ '''
+ request = self.build_request('/jobs')
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ for job in json.loads(response_data):
+ if job['name'] == job_name: return job['id']
+ print('CRITICAL No job with given name {} found'.format(job_name))
+ exit(1)
diff --git a/.gitlab/releaser/placeholder.py b/.gitlab/releaser/placeholder.py
new file mode 100644
index 000000000..a3624f0b3
--- /dev/null
+++ b/.gitlab/releaser/placeholder.py
@@ -0,0 +1,120 @@
+class PlaceHolder:
+ '''
+ Placeholder tags in Markdown texts.
+ '''
+ __PLACEHOLDER_PART = '<placeholder'
+ __PLACEHOLDER_START = '<placeholder content="{}">'
+ __PLACEHOLDER_STOP = '</placeholder>'
+ __PLACEHOLDER_FULL = '<placeholder content="{}" />'
+
+ def __init__(self, content_id):
+ '''
+ :param content_id: The identifier to be used for placeholder content.
+ :type content_id: str
+ '''
+ self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
+ self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
+ self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
+
+ def get_content(self, text):
+ '''
+ :param text: The text in which to extract content.
+ :type text: str
+ :return: The content between placeholder markers.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ text = text[pos + len(self.ph_start):]
+ pos = text.find(self.ph_stop)
+ if pos >= 0: text = text[:pos]
+ return text
+
+ def get_before(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content before (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if keep_mark: pos += len(self.ph_full)
+ text = text[:pos]
+ return text
+
+ def get_after(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content after (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if not keep_mark: pos += len(self.ph_full)
+ text = text[pos:]
+ return text
+
+ def replace_content(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been replaced.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ pos += len(self.ph_start)
+ text_before = text[:pos]
+ else:
+ pos = 0
+ text_before = ''
+ pos = text.find(self.ph_stop, pos)
+ if pos >= 0:
+ text_after = text[pos:]
+ else:
+ text_after = ''
+ return text_before + content + text_after
+
+ def insert_after(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been inserted.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0: pos += len(self.ph_full)
+ else: pos = 0
+ text_before = text[:pos]
+ text_after = text[pos:]
+ return text_before + content + text_after
+
+ def clear_all(text):
+ '''
+ Clear all placeholders from given text.
+ :param text: The text to clear.
+ :type text: str
+ :return: The clean text.
+ :rtype: str
+ '''
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
+ if pos < 0: break
+ end = text.find('>')
+ if end < 0: end = len(text)
+ text = text[:pos] + text[end + 1:]
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
+ if pos < 0: break
+ text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
+ return text
diff --git a/.gitlab/releaser/projectapi.py b/.gitlab/releaser/projectapi.py
new file mode 100644
index 000000000..eea07e769
--- /dev/null
+++ b/.gitlab/releaser/projectapi.py
@@ -0,0 +1,31 @@
+import os
+import urllib.request
+
+class ProjectApi:
+ '''
+ Gitlab API project access.
+ '''
+ __PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
+
+ def __init__(self, url=''):
+ '''
+ :param url: The URL portion to add to base project URL (if needed).
+ :type url: str
+ '''
+ self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
+ self.base_url += url
+ self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
+
+ def build_request(self, url='', **params):
+ '''
+ Create the request to send to project API.
+ :param url: The portion of URL to add to base URL (if needed).
+ :param params: The optional parameters.
+ :type url: str
+ :type params: dict
+ :return: The request, ready to be used.
+ :rtype: urllib.request.Request
+ '''
+ request = urllib.request.Request(self.base_url + url, **params)
+ request.add_header(*self.token)
+ return request
diff --git a/.gitlab/releaser/releasenote.py b/.gitlab/releaser/releasenote.py
new file mode 100644
index 000000000..c4fff9a75
--- /dev/null
+++ b/.gitlab/releaser/releasenote.py
@@ -0,0 +1,74 @@
+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseNote(ProjectApi):
+ '''
+ Release note API.
+ '''
+ __PH_TITLE = PlaceHolder('end-title')
+ __PH_NOTE = PlaceHolder('note')
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
+ self.message_read = False
+
+ def get_note(self):
+ '''
+ Get full release note.
+ :return: The note if it exists, None otherwise.
+ :rtype: str or None
+ '''
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ if data['release'] is None:
+ return None
+ else:
+ self.message_read = True
+ return data['release']['description']
+
+ def get_message(self):
+ '''
+ Get release message. Message is extracted from full note.
+ :return: The message if it exists, empty string otherwise.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ return ''
+ else:
+ return ReleaseNote.__PH_NOTE.get_content(data)
+
+ def get_note_body(self):
+ '''
+ Get release note body (without title). Body is extracted from full note.
+ :return: The body.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ print('CRITICAL No release information to publish')
+ exit(1)
+ return ReleaseNote.__PH_TITLE.get_after(data, True)
+
+ def send_note(self, note):
+ '''
+ Send the full release note. The current message should have been read
+ unless you are sure there are none.
+ :param note: The full note to send.
+ :type note: str
+ '''
+ method = 'PUT' if self.message_read else 'POST'
+ send_data = {
+ 'tag_name': os.environ['CI_COMMIT_TAG'],
+ 'description': note
+ }
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request('/release', data=send_data_serialized, method=method)
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/releaser.py b/.gitlab/releaser/releaser.py
new file mode 100644
index 000000000..125be7649
--- /dev/null
+++ b/.gitlab/releaser/releaser.py
@@ -0,0 +1,109 @@
+import glob
+import jinja2
+import json
+import os
+
+from binartifact import BinArtifact
+from job import Job
+from pipeline import Pipeline
+from placeholder import PlaceHolder
+from releasenote import ReleaseNote
+from releasewikipage import ReleaseWikiPage
+from sourceartifact import SourceArtifact
+from template import Template
+
+class Releaser:
+ '''
+ The main releaser class
+ '''
+
+ def __init__(self):
+ self.template = Template('release_template.md')
+ if 'RELEASE_BIN_DIR' in os.environ:
+ self.release_bin_dir = os.environ['RELEASE_BIN_DIR']
+ if not self.release_bin_dir.endswith('/'): self.release_bin_dir += '/'
+ else:
+ print('CRITICAL RELEASE_BIN_DIR environment variable not set')
+ exit(1)
+ if 'SOURCE_EXT' in os.environ:
+ self.source_ext = os.environ['SOURCE_EXT']
+ try:
+ self.source_ext = json.loads(self.source_ext)
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL SOURCE_EXT environment variable JSON probably malformed')
+ print('CRITICAL Correct : \'["zip","tar.gz"]\' ')
+ print('CRITICAL Not Correct: "[\'zip\',\'tar.gz\']" ')
+ exit(1)
+ else: self.source_ext = None
+
+ def release(self):
+ if self.source_ext is None:
+ self.publish_release()
+ else:
+ self.publish_prerelease()
+
+ def publish_prerelease(self):
+ '''
+ Main job to publish a pre-release.
+ '''
+ releaseNote = ReleaseNote()
+ current_message = releaseNote.get_message()
+ artifacts_list = []
+
+ # Get releases
+ artifacts_list += self._get_bin_artifacts()
+ artifacts_list.sort()
+ artifacts_list += list(map(lambda e: SourceArtifact(e), self.source_ext))
+
+ # Send result
+ note = self.template.render('notebody', {
+ 'current_message': current_message,
+ 'artifacts': list(map(lambda a: a.to_dict(), artifacts_list))
+ })
+ title_line = self.template.render('prerelease', {
+ 'tag': os.environ['CI_COMMIT_TAG'],
+ 'pipeline': os.environ['CI_PIPELINE_ID']
+ })
+ releaseNote.send_note(title_line + note)
+
+ print('Pre-release published')
+
+ def publish_release(self):
+ '''
+ Main job to publish the final release.
+ '''
+ # Change release note
+ releaseNote = ReleaseNote()
+ note = releaseNote.get_note_body()
+ title_line = self.template.render('release', {
+ 'tag': os.environ['CI_COMMIT_TAG'],
+ 'pipeline': os.environ['CI_PIPELINE_ID']
+ })
+ releaseNote.send_note(title_line + note)
+
+ # Update Wiki release page
+ wiki_page = ReleaseWikiPage(self.template)
+ wiki_page.add_release(os.environ['CI_COMMIT_TAG'], PlaceHolder.clear_all(note))
+ wiki_page.save()
+
+ # Keep artifacts
+ jobs = []
+ for artifact in self._get_bin_artifacts():
+ if not artifact.job in jobs:
+ jobs.append(artifact.job)
+ jobs = map(lambda j: Pipeline().find_job_id(j), jobs)
+ for job_id in jobs: Job(job_id).keep_artifacts()
+
+ print('Release published')
+
+ def _get_bin_artifacts(self):
+ '''
+ Get the binary artifacts for the current tag.
+ :return: The list of binary artifacts, based on found descriptions.
+ :rtype: list of BinArtifact
+ '''
+ DESC_EXT = '.desc'
+ artifacts = glob.glob('{}*{}'.format(self.release_bin_dir, DESC_EXT))
+ artifacts = map(lambda d: BinArtifact(self.release_bin_dir, d, DESC_EXT), artifacts)
+ artifacts = filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'], artifacts)
+ return list(artifacts)
diff --git a/.gitlab/releaser/releasewikipage.py b/.gitlab/releaser/releasewikipage.py
new file mode 100644
index 000000000..c76479610
--- /dev/null
+++ b/.gitlab/releaser/releasewikipage.py
@@ -0,0 +1,59 @@
+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseWikiPage(ProjectApi):
+ '''
+ Release Wiki page API.
+ '''
+ __PH_TAG = PlaceHolder('tag')
+ __PH_NOTE = PlaceHolder('note')
+ __PH_PREVIOUS = PlaceHolder('previous-beg')
+
+ def __init__(self, template):
+ '''
+ :param template: The template to use.
+ :type template: Template
+ '''
+ if not 'WIKI_RELEASE' in os.environ:
+ print('CRITICAL WIKI_RELEASE variable is not defined')
+ exit(1)
+ ProjectApi.__init__(self, '/wikis/{}'.format(os.environ['WIKI_RELEASE']))
+ self.template = template
+
+ # Query existing page
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ self.page_content = data['content']
+
+ def add_release(self, tag, note):
+ '''
+ Add the release to the Wiki page.
+ '''
+ prev_tag = ReleaseWikiPage.__PH_TAG.get_content(self.page_content)
+ prev_note = ReleaseWikiPage.__PH_NOTE.get_content(self.page_content)
+ self.page_content = ReleaseWikiPage.__PH_TAG.replace_content(self.page_content, tag)
+ self.page_content = ReleaseWikiPage.__PH_NOTE.replace_content(self.page_content, note)
+ previous = self.template.render('previouswiki', {
+ 'tag': prev_tag,
+ 'body': prev_note
+ })
+ self.page_content = ReleaseWikiPage.__PH_PREVIOUS.insert_after(
+ self.page_content, previous)
+
+ def save(self):
+ send_data = {
+ 'content': self.page_content,
+ 'format': 'markdown',
+ 'slug': os.environ['WIKI_RELEASE'],
+ 'title': os.environ['WIKI_RELEASE']
+ }
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request(data=send_data_serialized, method='PUT')
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/sourceartifact.py b/.gitlab/releaser/sourceartifact.py
new file mode 100644
index 000000000..b5eda1c81
--- /dev/null
+++ b/.gitlab/releaser/sourceartifact.py
@@ -0,0 +1,23 @@
+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class SourceArtifact(Artifact):
+ '''
+ A source artifact.
+ '''
+
+ def __init__(self, extention):
+ '''
+ :param extention: The extention of the source archive.
+ :type extention: str
+ '''
+ Artifact.__init__(self, 'archive.{}'.format(extention), 'Source code ({})'.format(extention), '', '', 'compression')
+
+ def _get_size(self):
+ return FSItemSize()
+
+ def _build_url(self):
+ return '{}/repository/{}/{}'.format(
+ os.environ['CI_PROJECT_URL'], os.environ['CI_COMMIT_TAG'], self.file_name)
diff --git a/.gitlab/releaser/template.py b/.gitlab/releaser/template.py
new file mode 100644
index 000000000..0c607a059
--- /dev/null
+++ b/.gitlab/releaser/template.py
@@ -0,0 +1,31 @@
+import jinja2
+import os
+
+class Template:
+ '''
+ Manages the template file. The template file is split into blocks.
+ '''
+ def __init__(self, fname):
+ '''
+ :param fname: The name of the template file.
+ :type fname: str
+ '''
+ path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ environment = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(path),
+ trim_blocks=True
+ )
+ self.template = environment.get_template(fname)
+
+ def render(self, block, params):
+ '''
+ Render a block from the template file.
+ :param block: The name of the block to render.
+ :param params: The parameters to be used in the block.
+ :type block: str
+ :type params: dict
+ :return: The rendered block.
+ :rtype: str
+ '''
+ context = self.template.new_context(params)
+ return jinja2.utils.concat(self.template.blocks[block](context))
--
GitLab