diff --git a/.gitignore b/.gitignore
index dcf30dda1096f9c692e98b34df94ad7a442c65eb..0a79bb0ce1ddaf1033f74641aff5c1ba8f6ce180 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,9 @@ gui/nw
vagrant/*.log
vagrant/duniter
+# Python compiled
+*.pyc
+
# Releases
/work
*.deb
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c53f5387130c38241f0a14b167f82db0f62a0dad..d75bd05da15a7dd66f0c65fee3450f34f25ab961 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -2,96 +2,102 @@ stages:
- github-sync
- build
- test
- - releases
- - releases-page
+ - package
+ - prerelease
+ - release
+
push_to_github:
- stage: github-sync
- variables:
- GIT_STRATEGY: none
- tags:
- - redshift
- script:
- - rm -rf ./*
- - rm -rf .git
- - git clone --mirror $CI_REPOSITORY_URL .
- - git remote add github $GITHUB_URL_AND_KEY
- - git config --global user.email "contact@duniter.org"
- - git config --global user.name "Duniter"
- # Job would fail if we don't remove refs about pull requests
- - bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
- - mv packed-refs-new packed-refs
- - bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
-
-build:
- stage: build
+ stage: github-sync
+ variables:
+ GIT_STRATEGY: none
+ tags:
+ - redshift
+ script:
+ - rm -rf ./*
+ - rm -rf .git
+ - git clone --mirror $CI_REPOSITORY_URL .
+ - git remote add github $GITHUB_URL_AND_KEY
+ - git config --global user.email "contact@duniter.org"
+ - git config --global user.name "Duniter"
+ # Job would fail if we don't remove refs about pull requests
+ - bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
+ - mv packed-refs-new packed-refs
+ - bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
+ only:
+ - nodes/typescript/duniter
+
+.nvm_env: &nvm_env
tags:
- redshift
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
+
+build:
+ <<: *nvm_env
+ stage: build
script:
- yarn
test:
+ <<: *nvm_env
stage: test
- tags:
- - redshift
- before_script:
- - export NVM_DIR="$HOME/.nvm"
- - . "$NVM_DIR/nvm.sh"
script:
- yarn
- yarn test
-releases:test:
- stage: releases
+.build_releases: &build_releases
+ stage: package
+ allow_failure: false
image: duniter/release-builder:v1.0.1
tags:
- redshift-duniter-builder
- variables:
- DAY: $(date +%Y%m%d)
- HOUR: $(date +%H%M)
- SEC: $(date +%S)
+ when: manual
+ artifacts:
+ paths: &releases_artifacts
+ - work/bin/
+
+releases:test:
+ <<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
artifacts:
- paths:
- - work/bin/
- expire_in: 8h
- when: manual
+ paths: *releases_artifacts
+ expire_in: 4h
except:
- tags
-
-releases:
- stage: releases
- image: duniter/release-builder:v1.0.1
- tags:
- - redshift-duniter-builder
+releases:x64:
+ <<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
artifacts:
- paths:
- - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb
- - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz
- - work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb
- expire_in: 8h
- when: manual
+ paths: *releases_artifacts
+ expire_in: 2 weeks
only:
- - tags
- - master
-
-releases-message:
- stage: releases-page
+ - tags
+
+.release_jobs: &release_jobs
image: tensorflow/tensorflow:latest-py3
tags:
- redshift-duniter-builder
- variables:
- JOB_ARTIFACTS: 'releases'
- EXPECTED_ARTIFACTS: '["work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb","work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz","work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb"]'
script:
- - python3 .gitlab/releaser.py
- when: manual
+ - python3 .gitlab/releaser
only:
- - tags
- - master
+ - tags
+
+prerelease:
+ <<: *release_jobs
+ stage: prerelease
+ variables:
+ RELEASE_BIN_DIR: work/bin/
+ SOURCE_EXT: '["tar.gz", "zip"]'
+
+publish:
+ <<: *release_jobs
+ stage: release
+ variables:
+ RELEASE_BIN_DIR: work/bin/
+ WIKI_RELEASE: Releases
+ allow_failure: false
+ when: manual
diff --git a/.gitlab/release_template.md b/.gitlab/release_template.md
index 99b23ff463aae901591987721b0db1f310710db9..1e04757fc7199f5096777d287fead0b89c997d33 100644
--- a/.gitlab/release_template.md
+++ b/.gitlab/release_template.md
@@ -1,9 +1,34 @@
+{% block prerelease %}
+# :gift: Pre-release
+
+[Go to Pipeline page :arrow_forward:](https://git.duniter.org/sveyret/duniter/pipelines/{{pipeline}})
+
+{% endblock %}
+
+{% block release %}
+# :white_check_mark: Release
+
+{% endblock %}
+
+{% block notebody %}
+<placeholder content="end-title" />
+<placeholder content="note">
{{current_message}}
+</placeholder>
-# Downloads
+## Downloads
+
+| Category | Arch | Type | Size | File |
+|----------|------|------|------|------|
{% for artifact in artifacts %}
-***
-[{{artifact.icon}} {{artifact.name}}]({{artifact.url}})
-_{{artifact.size}}_
-***
+| {{artifact.category}} | {{artifact.arch}} | {{artifact.type}} | {{artifact.size}} | [{{artifact.icon}} {{artifact.name}}]({{artifact.url}}) |
{% endfor %}
+{% endblock %}
+
+{% block previouswiki %}
+
+
+## {{tag}}
+
+{{body}}
+{% endblock %}
diff --git a/.gitlab/releaser.py b/.gitlab/releaser.py
deleted file mode 100644
index 833027fd0e589815d62f1d55308639c93eaa78d8..0000000000000000000000000000000000000000
--- a/.gitlab/releaser.py
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/usr/bin/python3
-'''
-This module is meant to overload the release note in gitlab for the current project.
-Expects to find in environment following variables:
- - CI_PROJECT_URL - Automatically set by gitlab-ci
- - CI_COMMIT_TAG - Automatically set by gitlab-ci
- - CI_PROJECT_ID - Automatically set by gitlab-ci
- - CI_COMMIT_TAG - Automatically set by gitlab-ci
- - RELEASER_TOKEN - Token used by technical user
- - JOB_ARTIFACTS - String containing job name containing all artifacts, to set manually
- - EXPECTED_ARTIFACTS - List containing all artifacts generated to set manually
-'''
-
-import math
-import urllib.request
-import urllib.error
-import json
-import os
-import jinja2
-
-def convert_size(size_bytes):
- '''Print proper size'''
- if size_bytes == 0:
- return '0B'
- size_name = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
- i = int(math.floor(math.log(size_bytes, 1024)))
- power = math.pow(1024, i)
- size = round(size_bytes / power, 2)
- return '%s %s' % (size, size_name[i])
-
-def get_current_message():
- '''Get current release message'''
- ci_project_id = os.environ['CI_PROJECT_ID']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- tag_url = 'https://git.duniter.org/api/v4/projects/'
- tag_url += ci_project_id
- tag_url += '/repository/tags/'
- tag_url += ci_commit_tag
- request = urllib.request.Request(tag_url)
- response = urllib.request.urlopen(request)
- response_data = response.read().decode()
- data = json.loads(response_data)
- if data['release'] is None:
- return False, ''
- else:
- return True, data['release']['description'].split('# Downloads')[0]
-
-def build_artifact_url(artifact, source):
- '''Given an artifact name, builds the url to download it'''
- job_artifacts = os.environ['JOB_ARTIFACTS']
- ci_project_url = os.environ['CI_PROJECT_URL']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- if source:
- source_url = ci_project_url
- source_url += '/repository/'
- source_url += ci_commit_tag
- source_url += '/archive.'
- source_url += artifact
- return source_url
- else:
- artifact_url = ci_project_url
- artifact_url += '/-/jobs/artifacts/'
- artifact_url += ci_commit_tag
- artifact_url += '/raw/'
- artifact_url += artifact
- artifact_url += '?job='
- artifact_url += job_artifacts
- return artifact_url
-
-def get_artifact_weight(location):
- '''Retrieve size of artifacts'''
- size = os.path.getsize(location)
- return convert_size(int(size))
-
-
-def build_compiled_message(current_message):
- '''Create a new release message using the release template'''
-
- expected_artifacts = os.environ['EXPECTED_ARTIFACTS']
- try:
- expected_artifacts = json.loads(expected_artifacts)
- except json.decoder.JSONDecodeError:
- print('CRITICAL EXPECTED_ARTIFACTS environment variable JSON probably malformed')
- print('CRITICAL Correct : \'["test_linux.txt","test_windows.txt"]\' ')
- print('CRITICAL Not Correct: "[\'test_linux.txt\',\'test_windows.txt\']" ')
- exit(1)
- artifacts_list = []
- for artifact in expected_artifacts:
- artifact_dict = {
- 'name': artifact.split('/')[-1],
- 'url': build_artifact_url(artifact, False),
- 'size': get_artifact_weight(artifact),
- 'icon': ':package:'
- }
- artifacts_list.append(artifact_dict)
-
- j2_env = jinja2.Environment(
- loader=jinja2.FileSystemLoader(
- os.path.dirname(os.path.abspath(__file__))
- ),
- trim_blocks=True
- )
- # pylint: disable=maybe-no-member
- template = j2_env.get_template('release_template.md')
- return template.render(
- current_message=current_message,
- artifacts=artifacts_list
- )
-
-
-def send_compiled_message(exists_release, compiled_message):
- '''Send to gitlab new message'''
- releaser_token = os.environ['RELEASER_TOKEN']
- ci_project_id = os.environ['CI_PROJECT_ID']
- ci_commit_tag = os.environ['CI_COMMIT_TAG']
- release_url = 'https://git.duniter.org/api/v4/projects/'
- release_url += ci_project_id
- release_url += '/repository/tags/'
- release_url += ci_commit_tag
- release_url += '/release'
- if exists_release:
- # We need to send a PUT request
- method = 'PUT'
- else:
- # We need to send a POST request
- method = 'POST'
- send_data = {
- 'tag_name':ci_commit_tag,
- 'description':compiled_message
- }
- send_data_serialized = json.dumps(send_data).encode('utf-8')
- request = urllib.request.Request(release_url, data=send_data_serialized, method=method)
- request.add_header('Private-Token', releaser_token)
- request.add_header('Content-Type', 'application/json')
- response = urllib.request.urlopen(request)
-
-def main():
- '''Execute main scenario'''
- exists_release, current_message = get_current_message()
- compiled_message = build_compiled_message(current_message)
- send_compiled_message(exists_release, compiled_message)
- print('Artifacts uploaded successfully')
-main()
diff --git a/.gitlab/releaser/__init__.py b/.gitlab/releaser/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..4f0a11e46162aa197c5f61768335d949955ae02b
--- /dev/null
+++ b/.gitlab/releaser/__init__.py
@@ -0,0 +1,12 @@
+'''
+This module is meant add release notes in gitlab for the current project.
+Expects to find in environment following variables:
+ - CI_PROJECT_URL - Automatically set by gitlab-ci
+ - CI_PROJECT_ID - Automatically set by gitlab-ci
+ - CI_COMMIT_TAG - Automatically set by gitlab-ci
+ - CI_PIPELINE_ID - Automatically set by gitlab-ci
+ - RELEASE_BIN_DIR - Directory where releases are to be found
+ - SOURCE_EXT - Source extensions (pre-release only)
+ - WIKI_RELEASE - Wiki page where releases are stored (release only)
+ - RELEASER_TOKEN - Token used by technical user
+'''
diff --git a/.gitlab/releaser/__main__.py b/.gitlab/releaser/__main__.py
new file mode 100644
index 0000000000000000000000000000000000000000..83a061b83f7f74905faeee23b9fa4702ffe3ab11
--- /dev/null
+++ b/.gitlab/releaser/__main__.py
@@ -0,0 +1,3 @@
+from releaser import Releaser
+
+Releaser().release()
diff --git a/.gitlab/releaser/artifact.py b/.gitlab/releaser/artifact.py
new file mode 100644
index 0000000000000000000000000000000000000000..e0d37f33a7d6896b471a708bf20a5c9e43ce8082
--- /dev/null
+++ b/.gitlab/releaser/artifact.py
@@ -0,0 +1,84 @@
+class Artifact:
+ '''
+ An artifact to be uploaded.
+ '''
+
+ def __init__(self, file_name, category, arch, dtype, icon):
+ '''
+ :param file_name: The name of the artifact file (may have directory).
+ :param category: The category (OS, distrib) for the artifact.
+ :param arch: The architecture name.
+ :param dtype: The delivery type (either server or desktop).
+ :param icon: The name of the icon to be used for artifact representation.
+ :type file_name: str
+ :type category: str
+ :type arch: str
+ :type dtype: str
+ :type icon: str
+ '''
+ self.file_name = file_name
+ self.category = category
+ self.arch = arch
+ self.dtype = dtype
+ self.icon = icon
+
+ def __lt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category < other.category or \
+ (self.category == other.category and self.arch < other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
+
+ def __le__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category <= other.category or \
+ (self.category == other.category and self.arch <= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
+
+ def __eq__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
+
+ def __ne__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
+
+ def __gt__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category > other.category or \
+ (self.category == other.category and self.arch > other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
+
+ def __ge__(self, other):
+ if not isinstance(other, Artifact): raise TypeError()
+ return self.category >= other.category or \
+ (self.category == other.category and self.arch >= other.arch) or \
+ (self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
+
+ def to_dict(self):
+ '''
+ :return: A dictionnary containing artifact data.
+ :rtype: dict
+ '''
+ return {
+ 'name': self.file_name.split('/')[-1],
+ 'category': self.category,
+ 'arch': self.arch,
+ 'type': self.dtype,
+ 'url': self._build_url(),
+ 'size': self._get_size(),
+ 'icon': ':{}:'.format(self.icon)
+ }
+
+ def _get_size(self):
+ '''
+ :return: The size of the artifact.
+ :rtype: FSItemSize
+ '''
+ raise NotImplementedError()
+
+ def _build_url(self):
+ '''
+ :return: The URL which can be used to get this artifact.
+ :rtype: str
+ '''
+ raise NotImplementedError()
diff --git a/.gitlab/releaser/binartifact.py b/.gitlab/releaser/binartifact.py
new file mode 100644
index 0000000000000000000000000000000000000000..45c617a6c07655796b5a7f087120e8dd27d2d071
--- /dev/null
+++ b/.gitlab/releaser/binartifact.py
@@ -0,0 +1,37 @@
+import json
+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class BinArtifact(Artifact):
+ '''
+ A binary artifact.
+ '''
+
+ def __init__(self, folder, desc_file, desc_ext):
+ '''
+ :param folder: The folder where files can be found.
+ :param desc_file: The name of the description file.
+ :param desc_ext: The extention of the description file.
+ :type folder: str
+ :type desc_file: str
+ :type desc_ext: str
+ '''
+ try:
+ description = json.load(open(desc_file))
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL Description file {} could not be read'.format(desc_file))
+ exit(1)
+
+ self.tag = description['version']
+ self.job = description['job']
+ file_name = desc_file[:-len(desc_ext)]
+ Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
+
+ def _get_size(self):
+ return FSItemSize(int(os.path.getsize(self.file_name)))
+
+ def _build_url(self):
+ return '{}/-/jobs/artifacts/{}/raw/{}?job={}'.format(
+ os.environ['CI_PROJECT_URL'], self.tag, self.file_name, self.job)
diff --git a/.gitlab/releaser/fsitemsize.py b/.gitlab/releaser/fsitemsize.py
new file mode 100644
index 0000000000000000000000000000000000000000..a6c8be232cdaf2ae00cebef732c225c27b427a3c
--- /dev/null
+++ b/.gitlab/releaser/fsitemsize.py
@@ -0,0 +1,28 @@
+import math
+
+class FSItemSize:
+ '''
+ The size of a file system item.
+ '''
+
+ def __init__(self, bsize = None):
+ '''
+ :param bsize: Size of item in bytes.
+ :type bsize: int
+ '''
+ self.bsize = bsize
+
+ def __str__(self):
+ '''
+ :return: Human readable size.
+ :rtype: str
+ '''
+ if self.bsize is None:
+ return '(unknown)'
+ elif self.bsize == 0:
+ return '0 B'
+ size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
+ i = int(math.floor(math.log(self.bsize, 1024)))
+ power = math.pow(1024, i)
+ size = round(self.bsize / power, 2)
+ return '{} {}'.format(size, size_name[i])
diff --git a/.gitlab/releaser/job.py b/.gitlab/releaser/job.py
new file mode 100644
index 0000000000000000000000000000000000000000..2dc55c08fcb14a76074b597b81f892a8aa339646
--- /dev/null
+++ b/.gitlab/releaser/job.py
@@ -0,0 +1,22 @@
+import urllib.request
+
+from projectapi import ProjectApi
+
+class Job(ProjectApi):
+ '''
+ Job data API.
+ '''
+
+ def __init__(self, job_id):
+ '''
+ :param job_id: The job id.
+ :type job_id: int
+ '''
+ ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
+
+ def keep_artifacts(self):
+ '''
+ Force artifacts to be kept forever.
+ '''
+ request = self.build_request('/artifacts/keep', method='POST')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/pipeline.py b/.gitlab/releaser/pipeline.py
new file mode 100644
index 0000000000000000000000000000000000000000..48e72216e90fcefc669b3c6c03338c64db5a2f69
--- /dev/null
+++ b/.gitlab/releaser/pipeline.py
@@ -0,0 +1,29 @@
+import json
+import os
+import urllib.request
+
+from projectapi import ProjectApi
+
+class Pipeline(ProjectApi):
+ '''
+ Pipeline data API.
+ '''
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/pipelines/{}'.format(os.environ['CI_PIPELINE_ID']))
+
+ def find_job_id(self, job_name):
+ '''
+ Find the id corresponding to given job name in the pipeline.
+ :param job_name: The job name.
+ :type job_name: str
+ :return: The identifier.
+ :rtype: int
+ '''
+ request = self.build_request('/jobs')
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ for job in json.loads(response_data):
+ if job['name'] == job_name: return job['id']
+ print('CRITICAL No job with given name {} found'.format(job_name))
+ exit(1)
diff --git a/.gitlab/releaser/placeholder.py b/.gitlab/releaser/placeholder.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3624f0b3b94ac494f3451820fbbb2ec127aba9a
--- /dev/null
+++ b/.gitlab/releaser/placeholder.py
@@ -0,0 +1,120 @@
+class PlaceHolder:
+ '''
+ Placeholder tags in Markdown texts.
+ '''
+ __PLACEHOLDER_PART = '<placeholder'
+ __PLACEHOLDER_START = '<placeholder content="{}">'
+ __PLACEHOLDER_STOP = '</placeholder>'
+ __PLACEHOLDER_FULL = '<placeholder content="{}" />'
+
+ def __init__(self, content_id):
+ '''
+ :param content_id: The identifier to be used for placeholder content.
+ :type content_id: str
+ '''
+ self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
+ self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
+ self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
+
+ def get_content(self, text):
+ '''
+ :param text: The text in which to extract content.
+ :type text: str
+ :return: The content between placeholder markers.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ text = text[pos + len(self.ph_start):]
+ pos = text.find(self.ph_stop)
+ if pos >= 0: text = text[:pos]
+ return text
+
+ def get_before(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content before (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if keep_mark: pos += len(self.ph_full)
+ text = text[:pos]
+ return text
+
+ def get_after(self, text, keep_mark=False):
+ '''
+ :param text: The text in which to extract content.
+ :param keep_mark: If true, the mark is kept in final text.
+ :type text: str
+ :type keep_mark: bool
+ :return: The content after (full) placeholder marker.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0:
+ if not keep_mark: pos += len(self.ph_full)
+ text = text[pos:]
+ return text
+
+ def replace_content(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been replaced.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_start)
+ if pos >= 0:
+ pos += len(self.ph_start)
+ text_before = text[:pos]
+ else:
+ pos = 0
+ text_before = ''
+ pos = text.find(self.ph_stop, pos)
+ if pos >= 0:
+ text_after = text[pos:]
+ else:
+ text_after = ''
+ return text_before + content + text_after
+
+ def insert_after(self, text, content):
+ '''
+ :param text: The text in which to extract content.
+ :param content: The new content to insert.
+ :type text: str
+ :type content: str
+ :return: The text where content has been inserted.
+ :rtype: str
+ '''
+ pos = text.find(self.ph_full)
+ if pos >= 0: pos += len(self.ph_full)
+ else: pos = 0
+ text_before = text[:pos]
+ text_after = text[pos:]
+ return text_before + content + text_after
+
+ def clear_all(text):
+ '''
+ Clear all placeholders from given text.
+ :param text: The text to clear.
+ :type text: str
+ :return: The clean text.
+ :rtype: str
+ '''
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
+ if pos < 0: break
+ end = text.find('>')
+ if end < 0: end = len(text)
+ text = text[:pos] + text[end + 1:]
+ while True:
+ pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
+ if pos < 0: break
+ text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
+ return text
diff --git a/.gitlab/releaser/projectapi.py b/.gitlab/releaser/projectapi.py
new file mode 100644
index 0000000000000000000000000000000000000000..eea07e769b95af92960e6538d7f2af60f3c699e8
--- /dev/null
+++ b/.gitlab/releaser/projectapi.py
@@ -0,0 +1,31 @@
+import os
+import urllib.request
+
+class ProjectApi:
+ '''
+ Gitlab API project access.
+ '''
+ __PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
+
+ def __init__(self, url=''):
+ '''
+ :param url: The URL portion to add to base project URL (if needed).
+ :type url: str
+ '''
+ self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
+ self.base_url += url
+ self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
+
+ def build_request(self, url='', **params):
+ '''
+ Create the request to send to project API.
+ :param url: The portion of URL to add to base URL (if needed).
+ :param params: The optional parameters.
+ :type url: str
+ :type params: dict
+ :return: The request, ready to be used.
+ :rtype: urllib.request.Request
+ '''
+ request = urllib.request.Request(self.base_url + url, **params)
+ request.add_header(*self.token)
+ return request
diff --git a/.gitlab/releaser/releasenote.py b/.gitlab/releaser/releasenote.py
new file mode 100644
index 0000000000000000000000000000000000000000..c4fff9a756273d38fed03907fb585e3db7fa90b1
--- /dev/null
+++ b/.gitlab/releaser/releasenote.py
@@ -0,0 +1,74 @@
+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseNote(ProjectApi):
+ '''
+ Release note API.
+ '''
+ __PH_TITLE = PlaceHolder('end-title')
+ __PH_NOTE = PlaceHolder('note')
+
+ def __init__(self):
+ ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
+ self.message_read = False
+
+ def get_note(self):
+ '''
+ Get full release note.
+ :return: The note if it exists, None otherwise.
+ :rtype: str or None
+ '''
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ if data['release'] is None:
+ return None
+ else:
+ self.message_read = True
+ return data['release']['description']
+
+ def get_message(self):
+ '''
+ Get release message. Message is extracted from full note.
+ :return: The message if it exists, empty string otherwise.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ return ''
+ else:
+ return ReleaseNote.__PH_NOTE.get_content(data)
+
+ def get_note_body(self):
+ '''
+ Get release note body (without title). Body is extracted from full note.
+ :return: The body.
+ :rtype: str
+ '''
+ data = self.get_note()
+ if data is None:
+ print('CRITICAL No release information to publish')
+ exit(1)
+ return ReleaseNote.__PH_TITLE.get_after(data, True)
+
+ def send_note(self, note):
+ '''
+ Send the full release note. The current message should have been read
+ unless you are sure there are none.
+ :param note: The full note to send.
+ :type note: str
+ '''
+ method = 'PUT' if self.message_read else 'POST'
+ send_data = {
+ 'tag_name': os.environ['CI_COMMIT_TAG'],
+ 'description': note
+ }
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request('/release', data=send_data_serialized, method=method)
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/releaser.py b/.gitlab/releaser/releaser.py
new file mode 100644
index 0000000000000000000000000000000000000000..125be7649ed844b5ba1709bfd89381edfb16058a
--- /dev/null
+++ b/.gitlab/releaser/releaser.py
@@ -0,0 +1,109 @@
+import glob
+import jinja2
+import json
+import os
+
+from binartifact import BinArtifact
+from job import Job
+from pipeline import Pipeline
+from placeholder import PlaceHolder
+from releasenote import ReleaseNote
+from releasewikipage import ReleaseWikiPage
+from sourceartifact import SourceArtifact
+from template import Template
+
+class Releaser:
+ '''
+ The main releaser class
+ '''
+
+ def __init__(self):
+ self.template = Template('release_template.md')
+ if 'RELEASE_BIN_DIR' in os.environ:
+ self.release_bin_dir = os.environ['RELEASE_BIN_DIR']
+ if not self.release_bin_dir.endswith('/'): self.release_bin_dir += '/'
+ else:
+ print('CRITICAL RELEASE_BIN_DIR environment variable not set')
+ exit(1)
+ if 'SOURCE_EXT' in os.environ:
+ self.source_ext = os.environ['SOURCE_EXT']
+ try:
+ self.source_ext = json.loads(self.source_ext)
+ except json.decoder.JSONDecodeError:
+ print('CRITICAL SOURCE_EXT environment variable JSON probably malformed')
+ print('CRITICAL Correct : \'["zip","tar.gz"]\' ')
+ print('CRITICAL Not Correct: "[\'zip\',\'tar.gz\']" ')
+ exit(1)
+ else: self.source_ext = None
+
+ def release(self):
+ if self.source_ext is None:
+ self.publish_release()
+ else:
+ self.publish_prerelease()
+
+ def publish_prerelease(self):
+ '''
+ Main job to publish a pre-release.
+ '''
+ releaseNote = ReleaseNote()
+ current_message = releaseNote.get_message()
+ artifacts_list = []
+
+ # Get releases
+ artifacts_list += self._get_bin_artifacts()
+ artifacts_list.sort()
+ artifacts_list += list(map(lambda e: SourceArtifact(e), self.source_ext))
+
+ # Send result
+ note = self.template.render('notebody', {
+ 'current_message': current_message,
+ 'artifacts': list(map(lambda a: a.to_dict(), artifacts_list))
+ })
+ title_line = self.template.render('prerelease', {
+ 'tag': os.environ['CI_COMMIT_TAG'],
+ 'pipeline': os.environ['CI_PIPELINE_ID']
+ })
+ releaseNote.send_note(title_line + note)
+
+ print('Pre-release published')
+
+ def publish_release(self):
+ '''
+ Main job to publish the final release.
+ '''
+ # Change release note
+ releaseNote = ReleaseNote()
+ note = releaseNote.get_note_body()
+ title_line = self.template.render('release', {
+ 'tag': os.environ['CI_COMMIT_TAG'],
+ 'pipeline': os.environ['CI_PIPELINE_ID']
+ })
+ releaseNote.send_note(title_line + note)
+
+ # Update Wiki release page
+ wiki_page = ReleaseWikiPage(self.template)
+ wiki_page.add_release(os.environ['CI_COMMIT_TAG'], PlaceHolder.clear_all(note))
+ wiki_page.save()
+
+ # Keep artifacts
+ jobs = []
+ for artifact in self._get_bin_artifacts():
+ if not artifact.job in jobs:
+ jobs.append(artifact.job)
+ jobs = map(lambda j: Pipeline().find_job_id(j), jobs)
+ for job_id in jobs: Job(job_id).keep_artifacts()
+
+ print('Release published')
+
+ def _get_bin_artifacts(self):
+ '''
+ Get the binary artifacts for the current tag.
+ :return: The list of binary artifacts, based on found descriptions.
+ :rtype: list of BinArtifact
+ '''
+ DESC_EXT = '.desc'
+ artifacts = glob.glob('{}*{}'.format(self.release_bin_dir, DESC_EXT))
+ artifacts = map(lambda d: BinArtifact(self.release_bin_dir, d, DESC_EXT), artifacts)
+ artifacts = filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'], artifacts)
+ return list(artifacts)
diff --git a/.gitlab/releaser/releasewikipage.py b/.gitlab/releaser/releasewikipage.py
new file mode 100644
index 0000000000000000000000000000000000000000..c76479610d4eba3ee0244df475f632c9fc9818fd
--- /dev/null
+++ b/.gitlab/releaser/releasewikipage.py
@@ -0,0 +1,59 @@
+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseWikiPage(ProjectApi):
+ '''
+ Release Wiki page API.
+ '''
+ __PH_TAG = PlaceHolder('tag')
+ __PH_NOTE = PlaceHolder('note')
+ __PH_PREVIOUS = PlaceHolder('previous-beg')
+
+ def __init__(self, template):
+ '''
+ :param template: The template to use.
+ :type template: Template
+ '''
+ if not 'WIKI_RELEASE' in os.environ:
+ print('CRITICAL WIKI_RELEASE variable is not defined')
+ exit(1)
+ ProjectApi.__init__(self, '/wikis/{}'.format(os.environ['WIKI_RELEASE']))
+ self.template = template
+
+ # Query existing page
+ request = self.build_request()
+ response = urllib.request.urlopen(request)
+ response_data = response.read().decode()
+ data = json.loads(response_data)
+ self.page_content = data['content']
+
+ def add_release(self, tag, note):
+ '''
+ Add the release to the Wiki page.
+ '''
+ prev_tag = ReleaseWikiPage.__PH_TAG.get_content(self.page_content)
+ prev_note = ReleaseWikiPage.__PH_NOTE.get_content(self.page_content)
+ self.page_content = ReleaseWikiPage.__PH_TAG.replace_content(self.page_content, tag)
+ self.page_content = ReleaseWikiPage.__PH_NOTE.replace_content(self.page_content, note)
+ previous = self.template.render('previouswiki', {
+ 'tag': prev_tag,
+ 'body': prev_note
+ })
+ self.page_content = ReleaseWikiPage.__PH_PREVIOUS.insert_after(
+ self.page_content, previous)
+
+ def save(self):
+ send_data = {
+ 'content': self.page_content,
+ 'format': 'markdown',
+ 'slug': os.environ['WIKI_RELEASE'],
+ 'title': os.environ['WIKI_RELEASE']
+ }
+ send_data_serialized = json.dumps(send_data).encode('utf-8')
+ request = self.build_request(data=send_data_serialized, method='PUT')
+ request.add_header('Content-Type', 'application/json')
+ urllib.request.urlopen(request)
diff --git a/.gitlab/releaser/sourceartifact.py b/.gitlab/releaser/sourceartifact.py
new file mode 100644
index 0000000000000000000000000000000000000000..b5eda1c81d26cbed6ced8339aff1001bdc562c70
--- /dev/null
+++ b/.gitlab/releaser/sourceartifact.py
@@ -0,0 +1,23 @@
+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class SourceArtifact(Artifact):
+ '''
+ A source artifact.
+ '''
+
+ def __init__(self, extention):
+ '''
+ :param extention: The extention of the source archive.
+ :type extention: str
+ '''
+ Artifact.__init__(self, 'archive.{}'.format(extention), 'Source code ({})'.format(extention), '', '', 'compression')
+
+ def _get_size(self):
+ return FSItemSize()
+
+ def _build_url(self):
+ return '{}/repository/{}/{}'.format(
+ os.environ['CI_PROJECT_URL'], os.environ['CI_COMMIT_TAG'], self.file_name)
diff --git a/.gitlab/releaser/template.py b/.gitlab/releaser/template.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c607a059dade516d96db0a0a89b8d43b82921e6
--- /dev/null
+++ b/.gitlab/releaser/template.py
@@ -0,0 +1,31 @@
+import jinja2
+import os
+
+class Template:
+ '''
+ Manages the template file. The template file is split into blocks.
+ '''
+ def __init__(self, fname):
+ '''
+ :param fname: The name of the template file.
+ :type fname: str
+ '''
+ path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ environment = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(path),
+ trim_blocks=True
+ )
+ self.template = environment.get_template(fname)
+
+ def render(self, block, params):
+ '''
+ Render a block from the template file.
+ :param block: The name of the block to render.
+ :param params: The parameters to be used in the block.
+ :type block: str
+ :type params: dict
+ :return: The rendered block.
+ :rtype: str
+ '''
+ context = self.template.new_context(params)
+ return jinja2.utils.concat(self.template.blocks[block](context))
diff --git a/release/arch/linux/build-lin.sh b/release/arch/linux/build-lin.sh
index 585ee00195f9cff1ab89d817584af3e7e4c1bf3f..b06938b45714bce9f5f8c64cc4007c097bbe4c39 100644
--- a/release/arch/linux/build-lin.sh
+++ b/release/arch/linux/build-lin.sh
@@ -11,6 +11,105 @@ else
exit 1
fi
+# ---------
+# Functions
+# ---------
+
+# Copy nw.js compiled module released library to node libraries.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy() {
+ [[ -z ${1} ]] && exit 1
+ cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
+ lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
+}
+
+# Copy nw.js compiled module library to node libraries, prefixing with node_.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy_node() {
+ [[ -z ${1} ]] && exit 1
+ cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
+ lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
+}
+
+# Compile the module with nw.js.
+# -
+# Parameters:
+# 1. Module name.
+# 2. Action to be done to module after compilation, if needed.
+nw_compile() {
+ [[ -z ${1} ]] && exit 1
+ cd ${1} || exit 1
+ node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
+ node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
+ [[ -z ${2} ]] || ${2} ${1}
+ cd ..
+}
+
+# Create description.
+# -
+# Parameters:
+# 1. Initial file name.
+# 2. Building type (either “desktop” or “server”).
+# 3. Category (OS, distribution).
+create_desc() {
+ cat >"${1}".desc <<-EOF
+ {
+ "version": "${DUNITER_TAG}",
+ "job": "${CI_JOB_NAME}",
+ "type": "${2^}",
+ "category": "${3}",
+ "arch": "x64"
+ }
+ EOF
+}
+
+# Desktop specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_desktop() {
+ cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
+}
+
+# Server specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_server() {
+ mkdir -p "${1}/lib/systemd/system" || exit 1
+ cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
+}
+
+# Debian package building.
+# -
+# Parameters:
+# 1. Building type (either “desktop” or “server”).
+# 2. Debian package name.
+build_deb_pack() {
+ rm -rf "${RELEASES}/duniter-x64"
+ mkdir "${RELEASES}/duniter-x64" || exit 1
+ cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
+ build_extra_${1} "${RELEASES}/duniter-x64"
+ mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
+ chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
+ chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
+ sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+ cd "${RELEASES}/${1}_/"
+ zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
+
+ sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+ cd "${RELEASES}"
+ fakeroot dpkg-deb --build duniter-x64 || exit 1
+ mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
+ create_desc "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" "${1}" "Linux (Ubuntu/Debian)"
+}
+
# -----------
# Prepare
# -----------
@@ -43,7 +142,7 @@ RELEASES="${WORK}/releases"
BIN="${WORK}/bin"
mkdir -p "${DOWNLOADS}" "${RELEASES}" "${BIN}" || exit 1
-rm -rf "${BIN}/"*.{deb,tar.gz} # Clean up
+rm -rf "${BIN}/"*.{deb,tar.gz}{,.desc} # Clean up
# -----------
# Downloads
@@ -83,27 +182,6 @@ cp -r "${RELEASES}/duniter" "${RELEASES}/server_" || exit 1
# Build Desktop version against nw.js
# -------------------------------------
-nw_copy() {
- [[ -z ${1} ]] && exit 1
- cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
- lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
-}
-
-nw_copy_node() {
- [[ -z ${1} ]] && exit 1
- cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
- lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
-}
-
-nw_compile() {
- [[ -z ${1} ]] && exit 1
- cd ${1} || exit 1
- node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
- node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
- [[ -z ${2} ]] || ${2} ${1}
- cd ..
-}
-
echo "${NW_RELEASE}"
# FIX: bug of nw.js, we need to patch first.
@@ -162,46 +240,11 @@ cp -r "${DOWNLOADS}/node-${NVER}-linux-x64" "${RELEASES}/server_/node" || exit 1
cd "${RELEASES}/desktop_"
tar czf "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" * || exit 1
+create_desc "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" "Desktop" "Linux (generic)"
# -----------------------
# Build Debian packages
# -----------------------
-# Parameters
-# 1: Building directory.
-build_extra_desktop() {
- cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
-}
-
-# Parameters
-# 1: Building directory.
-build_extra_server() {
- mkdir -p "${1}/lib/systemd/system" || exit 1
- cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
-}
-
-# Parameters
-# 1: either "server" or "desktop".
-# 2: package name for Debian.
-build_deb_pack() {
- rm -rf "${RELEASES}/duniter-x64"
- mkdir "${RELEASES}/duniter-x64" || exit 1
- cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
- build_extra_${1} "${RELEASES}/duniter-x64"
- mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
- chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
- chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
- sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
- cd "${RELEASES}/${1}_/"
- zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
-
- sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
- cd "${RELEASES}"
- fakeroot dpkg-deb --build duniter-x64 || exit 1
- mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
-}
-
build_deb_pack desktop duniter-desktop
build_deb_pack server duniter