Merge branch 'AutoRelease' into '1.6'

Automatic release management

See merge request !1240

.gitignore

@@ -18,6 +18,9 @@ gui/nw
 vagrant/*.log
 vagrant/duniter
 
+# Python compiled
+*.pyc
+
 # Releases
 /work
 *.deb

.gitlab-ci.yml

@@ -2,8 +2,10 @@ stages:
   - github-sync
   - build
   - test
-  - releases
-  - releases-page
+  - package
+  - prerelease
+  - release
+
 push_to_github:
   stage: github-sync
   variables:
@@ -21,77 +23,81 @@ push_to_github:
     - bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
     - mv packed-refs-new packed-refs
     - bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
+  only:
+    - nodes/typescript/duniter
 
-build:
-  stage: build
+.nvm_env: &nvm_env
   tags:
     - redshift
   before_script:
     - export NVM_DIR="$HOME/.nvm"
     - . "$NVM_DIR/nvm.sh"
+  
+build:
+  <<: *nvm_env
+  stage: build
   script:
     - yarn
 
 test:
+  <<: *nvm_env
   stage: test
-  tags:
-    - redshift
-  before_script:
-    - export NVM_DIR="$HOME/.nvm"
-    - . "$NVM_DIR/nvm.sh"
   script:
     - yarn
     - yarn test
 
-releases:test:
-  stage: releases
+.build_releases: &build_releases
+  stage: package
+  allow_failure: false
   image: duniter/release-builder:v1.0.1
   tags:
     - redshift-duniter-builder
-  variables:
-    DAY: $(date +%Y%m%d)
-    HOUR: $(date +%H%M)
-    SEC: $(date +%S)
+  when: manual
+  artifacts:
+    paths: &releases_artifacts
+      - work/bin/
+
+releases:test:
+  <<: *build_releases
   script:
     - bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
   artifacts:
-    paths:
-      - work/bin/
-    expire_in: 8h
-  when: manual
+    paths: *releases_artifacts
+    expire_in: 4h
   except:
     - tags
 
-
-releases:
-  stage: releases
-  image: duniter/release-builder:v1.0.1
-  tags:
-    - redshift-duniter-builder
+releases:x64:
+  <<: *build_releases
   script:
     - bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
   artifacts:
-    paths:
-      - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb
-      - work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz
-      - work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb
-    expire_in: 8h
-  when: manual
+    paths: *releases_artifacts
+    expire_in: 2 weeks
   only:
     - tags
-  - master
 
-releases-message:
-  stage: releases-page
+.release_jobs: &release_jobs
   image: tensorflow/tensorflow:latest-py3
   tags:
     - redshift-duniter-builder
-  variables:
-    JOB_ARTIFACTS: 'releases'
-    EXPECTED_ARTIFACTS: '["work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb","work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz","work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb"]'
   script:
-    - python3 .gitlab/releaser.py
-  when: manual
+    - python3 .gitlab/releaser
   only:
     - tags
-  - master
+
+prerelease:
+  <<: *release_jobs
+  stage: prerelease
+  variables:
+    RELEASE_BIN_DIR: work/bin/
+    SOURCE_EXT: '["tar.gz", "zip"]'
+
+publish:
+  <<: *release_jobs
+  stage: release
+  variables:
+    RELEASE_BIN_DIR: work/bin/
+    WIKI_RELEASE: Releases
+  allow_failure: false
+  when: manual

.gitlab/release_template.md

+{% block prerelease %}
+# :gift: Pre-release
+
+[Go to Pipeline page :arrow_forward:](https://git.duniter.org/sveyret/duniter/pipelines/{{pipeline}})
+
+{% endblock %}
+
+{% block release %}
+# :white_check_mark: Release
+
+{% endblock %}
+
+{% block notebody %}
+<placeholder content="end-title" />
+<placeholder content="note">
 {{current_message}}
+</placeholder>
 
-# Downloads
+## Downloads
+
+| Category | Arch | Type | Size | File |
+|----------|------|------|------|------|
 {% for artifact in artifacts %}
-***
-[{{artifact.icon}} {{artifact.name}}]({{artifact.url}})  
-_{{artifact.size}}_
-***
+| {{artifact.category}} | {{artifact.arch}} | {{artifact.type}} | {{artifact.size}} | [{{artifact.icon}} {{artifact.name}}]({{artifact.url}}) |
 {% endfor %}
+{% endblock %}
+
+{% block previouswiki %}
+
+
+## {{tag}}
+
+{{body}}
+{% endblock %}

.gitlab/releaser.py

-#!/usr/bin/python3
-'''
-This module is meant to overload the release note in gitlab for the current project.
-Expects to find in environment following variables:
-  - CI_PROJECT_URL - Automatically set by gitlab-ci
-  - CI_COMMIT_TAG - Automatically set by gitlab-ci
-  - CI_PROJECT_ID - Automatically set by gitlab-ci
-  - CI_COMMIT_TAG - Automatically set by gitlab-ci
-  - RELEASER_TOKEN - Token used by technical user
-  - JOB_ARTIFACTS - String containing job name containing all artifacts, to set manually
-  - EXPECTED_ARTIFACTS - List containing all artifacts generated to set manually
-'''
-
-import math
-import urllib.request
-import urllib.error
-import json
-import os
-import jinja2
-
-def convert_size(size_bytes):
-    '''Print proper size'''
-    if size_bytes == 0:
-        return '0B'
-    size_name = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
-    i = int(math.floor(math.log(size_bytes, 1024)))
-    power = math.pow(1024, i)
-    size = round(size_bytes / power, 2)
-    return '%s %s' % (size, size_name[i])
-
-def get_current_message():
-    '''Get current release message'''
-    ci_project_id = os.environ['CI_PROJECT_ID']
-    ci_commit_tag = os.environ['CI_COMMIT_TAG']
-    tag_url = 'https://git.duniter.org/api/v4/projects/'
-    tag_url += ci_project_id
-    tag_url += '/repository/tags/'
-    tag_url += ci_commit_tag
-    request = urllib.request.Request(tag_url)
-    response = urllib.request.urlopen(request)
-    response_data = response.read().decode()
-    data = json.loads(response_data)
-    if data['release'] is None:
-        return False, ''
-    else:
-        return True, data['release']['description'].split('# Downloads')[0]
-
-def build_artifact_url(artifact, source):
-    '''Given an artifact name, builds the url to download it'''
-    job_artifacts = os.environ['JOB_ARTIFACTS']
-    ci_project_url = os.environ['CI_PROJECT_URL']
-    ci_commit_tag = os.environ['CI_COMMIT_TAG']
-    if source:
-        source_url = ci_project_url
-        source_url += '/repository/'
-        source_url += ci_commit_tag
-        source_url += '/archive.'
-        source_url += artifact
-        return source_url
-    else:
-        artifact_url = ci_project_url
-        artifact_url += '/-/jobs/artifacts/'
-        artifact_url += ci_commit_tag
-        artifact_url += '/raw/'
-        artifact_url += artifact
-        artifact_url += '?job='
-        artifact_url += job_artifacts
-        return artifact_url
-
-def get_artifact_weight(location):
-    '''Retrieve size of artifacts'''
-    size = os.path.getsize(location)
-    return convert_size(int(size))
-
-
-def build_compiled_message(current_message):
-    '''Create a new release message using the release template'''
-
-    expected_artifacts = os.environ['EXPECTED_ARTIFACTS']
-    try:
-        expected_artifacts = json.loads(expected_artifacts)
-    except json.decoder.JSONDecodeError:
-        print('CRITICAL EXPECTED_ARTIFACTS environment variable JSON probably malformed')
-        print('CRITICAL Correct : \'["test_linux.txt","test_windows.txt"]\' ')
-        print('CRITICAL Not Correct: "[\'test_linux.txt\',\'test_windows.txt\']" ')
-        exit(1)
-    artifacts_list = []
-    for artifact in expected_artifacts:
-        artifact_dict = {
-            'name': artifact.split('/')[-1],
-            'url': build_artifact_url(artifact, False),
-            'size': get_artifact_weight(artifact),
-            'icon': ':package:'
-        }
-        artifacts_list.append(artifact_dict)
-
-    j2_env = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(
-            os.path.dirname(os.path.abspath(__file__))
-            ),
-        trim_blocks=True
-        )
-    # pylint: disable=maybe-no-member
-    template = j2_env.get_template('release_template.md')
-    return template.render(
-        current_message=current_message,
-        artifacts=artifacts_list
-    )
-
-
-def send_compiled_message(exists_release, compiled_message):
-    '''Send to gitlab new message'''
-    releaser_token = os.environ['RELEASER_TOKEN']
-    ci_project_id = os.environ['CI_PROJECT_ID']
-    ci_commit_tag = os.environ['CI_COMMIT_TAG']
-    release_url = 'https://git.duniter.org/api/v4/projects/'
-    release_url += ci_project_id
-    release_url += '/repository/tags/'
-    release_url += ci_commit_tag
-    release_url += '/release'
-    if exists_release:
-        # We need to send a PUT request
-        method = 'PUT'
-    else:
-        # We need to send a POST request
-        method = 'POST'
-    send_data = {
-        'tag_name':ci_commit_tag,
-        'description':compiled_message
-        }
-    send_data_serialized = json.dumps(send_data).encode('utf-8')
-    request = urllib.request.Request(release_url, data=send_data_serialized, method=method)
-    request.add_header('Private-Token', releaser_token)
-    request.add_header('Content-Type', 'application/json')
-    response = urllib.request.urlopen(request)
-
-def main():
-    '''Execute main scenario'''
-    exists_release, current_message = get_current_message()
-    compiled_message = build_compiled_message(current_message)
-    send_compiled_message(exists_release, compiled_message)
-    print('Artifacts uploaded successfully')
-main()

.gitlab/releaser/__init__.py

+'''
+This module is meant add release notes in gitlab for the current project.
+Expects to find in environment following variables:
+  - CI_PROJECT_URL - Automatically set by gitlab-ci
+  - CI_PROJECT_ID - Automatically set by gitlab-ci
+  - CI_COMMIT_TAG - Automatically set by gitlab-ci
+  - CI_PIPELINE_ID - Automatically set by gitlab-ci
+  - RELEASE_BIN_DIR - Directory where releases are to be found
+  - SOURCE_EXT - Source extensions (pre-release only)
+  - WIKI_RELEASE - Wiki page where releases are stored (release only)
+  - RELEASER_TOKEN - Token used by technical user
+'''

.gitlab/releaser/__main__.py

+from releaser import Releaser
+
+Releaser().release()

.gitlab/releaser/artifact.py

+class Artifact:
+    '''
+    An artifact to be uploaded.
+    '''
+
+    def __init__(self, file_name, category, arch, dtype, icon):
+        '''
+        :param file_name: The name of the artifact file (may have directory).
+        :param category: The category (OS, distrib) for the artifact.
+        :param arch: The architecture name.
+        :param dtype: The delivery type (either server or desktop).
+        :param icon: The name of the icon to be used for artifact representation.
+        :type file_name: str
+        :type category: str
+        :type arch: str
+        :type dtype: str
+        :type icon: str
+        '''
+        self.file_name = file_name
+        self.category = category
+        self.arch = arch
+        self.dtype = dtype
+        self.icon = icon
+
+    def __lt__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category < other.category or \
+            (self.category == other.category and self.arch < other.arch) or \
+            (self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
+
+    def __le__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category <= other.category or \
+            (self.category == other.category and self.arch <= other.arch) or \
+            (self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
+
+    def __eq__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
+
+    def __ne__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
+
+    def __gt__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category > other.category or \
+            (self.category == other.category and self.arch > other.arch) or \
+            (self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
+
+    def __ge__(self, other):
+        if not isinstance(other, Artifact): raise TypeError()
+        return self.category >= other.category or \
+            (self.category == other.category and self.arch >= other.arch) or \
+            (self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
+
+    def to_dict(self):
+        '''
+        :return: A dictionnary containing artifact data.
+        :rtype: dict
+        '''
+        return {
+            'name': self.file_name.split('/')[-1],
+            'category': self.category,
+            'arch': self.arch,
+            'type': self.dtype,
+            'url': self._build_url(),
+            'size': self._get_size(),
+            'icon': ':{}:'.format(self.icon)
+        }
+
+    def _get_size(self):
+        '''
+        :return: The size of the artifact.
+        :rtype: FSItemSize
+        '''
+        raise NotImplementedError()
+
+    def _build_url(self):
+        '''
+        :return: The URL which can be used to get this artifact.
+        :rtype: str
+        '''
+        raise NotImplementedError()

.gitlab/releaser/binartifact.py

+import json
+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class BinArtifact(Artifact):
+    '''
+    A binary artifact.
+    '''
+
+    def __init__(self, folder, desc_file, desc_ext):
+        '''
+        :param folder: The folder where files can be found.
+        :param desc_file: The name of the description file.
+        :param desc_ext: The extention of the description file.
+        :type folder: str
+        :type desc_file: str
+        :type desc_ext: str
+        '''
+        try:
+            description = json.load(open(desc_file))
+        except json.decoder.JSONDecodeError:
+            print('CRITICAL Description file {} could not be read'.format(desc_file))
+            exit(1)
+
+        self.tag = description['version']
+        self.job = description['job']
+        file_name = desc_file[:-len(desc_ext)]
+        Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
+
+    def _get_size(self):
+        return FSItemSize(int(os.path.getsize(self.file_name)))
+
+    def _build_url(self):
+        return '{}/-/jobs/artifacts/{}/raw/{}?job={}'.format(
+            os.environ['CI_PROJECT_URL'], self.tag, self.file_name, self.job)

.gitlab/releaser/fsitemsize.py

+import math
+
+class FSItemSize:
+    '''
+    The size of a file system item.
+    '''
+
+    def __init__(self, bsize = None):
+        '''
+        :param bsize: Size of item in bytes.
+        :type bsize: int
+        '''
+        self.bsize = bsize
+
+    def __str__(self):
+        '''
+        :return: Human readable size.
+        :rtype: str
+        '''
+        if self.bsize is None:
+            return '(unknown)'
+        elif self.bsize == 0:
+            return '0 B'
+        size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
+        i = int(math.floor(math.log(self.bsize, 1024)))
+        power = math.pow(1024, i)
+        size = round(self.bsize / power, 2)
+        return '{} {}'.format(size, size_name[i])

.gitlab/releaser/job.py

+import urllib.request
+
+from projectapi import ProjectApi
+
+class Job(ProjectApi):
+    '''
+    Job data API.
+    '''
+
+    def __init__(self, job_id):
+        '''
+        :param job_id: The job id.
+        :type job_id: int
+        '''
+        ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
+
+    def keep_artifacts(self):
+        '''
+        Force artifacts to be kept forever.
+        '''
+        request = self.build_request('/artifacts/keep', method='POST')
+        urllib.request.urlopen(request)

.gitlab/releaser/pipeline.py

+import json
+import os
+import urllib.request
+
+from projectapi import ProjectApi
+
+class Pipeline(ProjectApi):
+    '''
+    Pipeline data API.
+    '''
+
+    def __init__(self):
+        ProjectApi.__init__(self, '/pipelines/{}'.format(os.environ['CI_PIPELINE_ID']))
+
+    def find_job_id(self, job_name):
+        '''
+        Find the id corresponding to given job name in the pipeline.
+        :param job_name: The job name.
+        :type job_name: str
+        :return: The identifier.
+        :rtype: int
+        '''
+        request = self.build_request('/jobs')
+        response = urllib.request.urlopen(request)
+        response_data = response.read().decode()
+        for job in json.loads(response_data):
+            if job['name'] == job_name: return job['id']
+        print('CRITICAL No job with given name {} found'.format(job_name))
+        exit(1)

.gitlab/releaser/placeholder.py

+class PlaceHolder:
+    '''
+    Placeholder tags in Markdown texts.
+    '''
+    __PLACEHOLDER_PART = '<placeholder'
+    __PLACEHOLDER_START = '<placeholder content="{}">'
+    __PLACEHOLDER_STOP = '</placeholder>'
+    __PLACEHOLDER_FULL = '<placeholder content="{}" />'
+
+    def __init__(self, content_id):
+        '''
+        :param content_id: The identifier to be used for placeholder content.
+        :type content_id: str
+        '''
+        self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
+        self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
+        self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
+
+    def get_content(self, text):
+        '''
+        :param text: The text in which to extract content.
+        :type text: str
+        :return: The content between placeholder markers.
+        :rtype: str
+        '''
+        pos = text.find(self.ph_start)
+        if pos >= 0:
+            text = text[pos + len(self.ph_start):]
+            pos = text.find(self.ph_stop)
+            if pos >= 0: text = text[:pos]
+        return text
+
+    def get_before(self, text, keep_mark=False):
+        '''
+        :param text: The text in which to extract content.
+        :param keep_mark: If true, the mark is kept in final text.
+        :type text: str
+        :type keep_mark: bool
+        :return: The content before (full) placeholder marker.
+        :rtype: str
+        '''
+        pos = text.find(self.ph_full)
+        if pos >= 0:
+            if keep_mark: pos += len(self.ph_full)
+            text = text[:pos]
+        return text
+
+    def get_after(self, text, keep_mark=False):
+        '''
+        :param text: The text in which to extract content.
+        :param keep_mark: If true, the mark is kept in final text.
+        :type text: str
+        :type keep_mark: bool
+        :return: The content after (full) placeholder marker.
+        :rtype: str
+        '''
+        pos = text.find(self.ph_full)
+        if pos >= 0:
+            if not keep_mark: pos += len(self.ph_full)
+            text = text[pos:]
+        return text
+
+    def replace_content(self, text, content):
+        '''
+        :param text: The text in which to extract content.
+        :param content: The new content to insert.
+        :type text: str
+        :type content: str
+        :return: The text where content has been replaced.
+        :rtype: str
+        '''
+        pos = text.find(self.ph_start)
+        if pos >= 0:
+            pos += len(self.ph_start)
+            text_before = text[:pos]
+        else:
+            pos = 0
+            text_before = ''
+        pos = text.find(self.ph_stop, pos)
+        if pos >= 0:
+            text_after = text[pos:]
+        else:
+            text_after = ''
+        return text_before + content + text_after
+
+    def insert_after(self, text, content):
+        '''
+        :param text: The text in which to extract content.
+        :param content: The new content to insert.
+        :type text: str
+        :type content: str
+        :return: The text where content has been inserted.
+        :rtype: str
+        '''
+        pos = text.find(self.ph_full)
+        if pos >= 0: pos += len(self.ph_full)
+        else: pos = 0
+        text_before = text[:pos]
+        text_after = text[pos:]
+        return text_before + content + text_after
+
+    def clear_all(text):
+        '''
+        Clear all placeholders from given text.
+        :param text: The text to clear.
+        :type text: str
+        :return: The clean text.
+        :rtype: str
+        '''
+        while True:
+            pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
+            if pos < 0: break
+            end = text.find('>')
+            if end < 0: end = len(text)
+            text = text[:pos] + text[end + 1:]
+        while True:
+            pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
+            if pos < 0: break
+            text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
+        return text

.gitlab/releaser/projectapi.py

+import os
+import urllib.request
+
+class ProjectApi:
+    '''
+    Gitlab API project access.
+    '''
+    __PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
+
+    def __init__(self, url=''):
+        '''
+        :param url: The URL portion to add to base project URL (if needed).
+        :type url: str
+        '''
+        self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
+        self.base_url += url
+        self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
+
+    def build_request(self, url='', **params):
+        '''
+        Create the request to send to project API.
+        :param url: The portion of URL to add to base URL (if needed).
+        :param params: The optional parameters.
+        :type url: str
+        :type params: dict
+        :return: The request, ready to be used.
+        :rtype: urllib.request.Request
+        '''
+        request = urllib.request.Request(self.base_url + url, **params)
+        request.add_header(*self.token)
+        return request

.gitlab/releaser/releasenote.py

+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseNote(ProjectApi):
+    '''
+    Release note API.
+    '''
+    __PH_TITLE = PlaceHolder('end-title')
+    __PH_NOTE = PlaceHolder('note')
+
+    def __init__(self):
+        ProjectApi.__init__(self, '/repository/tags/{}'.format(os.environ['CI_COMMIT_TAG']))
+        self.message_read = False
+
+    def get_note(self):
+        '''
+        Get full release note.
+        :return: The note if it exists, None otherwise.
+        :rtype: str or None
+        '''
+        request = self.build_request()
+        response = urllib.request.urlopen(request)
+        response_data = response.read().decode()
+        data = json.loads(response_data)
+        if data['release'] is None:
+            return None
+        else:
+            self.message_read = True
+            return data['release']['description']
+
+    def get_message(self):
+        '''
+        Get release message. Message is extracted from full note.
+        :return: The message if it exists, empty string otherwise.
+        :rtype: str
+        '''
+        data = self.get_note()
+        if data is None:
+            return ''
+        else:
+            return ReleaseNote.__PH_NOTE.get_content(data)
+
+    def get_note_body(self):
+        '''
+        Get release note body (without title). Body is extracted from full note.
+        :return: The body.
+        :rtype: str
+        '''
+        data = self.get_note()
+        if data is None:
+            print('CRITICAL No release information to publish')
+            exit(1)
+        return ReleaseNote.__PH_TITLE.get_after(data, True)
+
+    def send_note(self, note):
+        '''
+        Send the full release note. The current message should have been read
+        unless you are sure there are none.
+        :param note: The full note to send.
+        :type note: str
+        '''
+        method = 'PUT' if self.message_read else 'POST'
+        send_data = {
+            'tag_name': os.environ['CI_COMMIT_TAG'],
+            'description': note
+        }
+        send_data_serialized = json.dumps(send_data).encode('utf-8')
+        request = self.build_request('/release', data=send_data_serialized, method=method)
+        request.add_header('Content-Type', 'application/json')
+        urllib.request.urlopen(request)

.gitlab/releaser/releaser.py

+import glob
+import jinja2
+import json
+import os
+
+from binartifact import BinArtifact
+from job import Job
+from pipeline import Pipeline
+from placeholder import PlaceHolder
+from releasenote import ReleaseNote
+from releasewikipage import ReleaseWikiPage
+from sourceartifact import SourceArtifact
+from template import Template
+
+class Releaser:
+    '''
+    The main releaser class
+    '''
+
+    def __init__(self):
+        self.template = Template('release_template.md')
+        if 'RELEASE_BIN_DIR' in os.environ:
+            self.release_bin_dir = os.environ['RELEASE_BIN_DIR']
+            if not self.release_bin_dir.endswith('/'): self.release_bin_dir += '/'
+        else:
+            print('CRITICAL RELEASE_BIN_DIR environment variable not set')
+            exit(1)
+        if 'SOURCE_EXT' in os.environ:
+            self.source_ext = os.environ['SOURCE_EXT']
+            try:
+                self.source_ext = json.loads(self.source_ext)
+            except json.decoder.JSONDecodeError:
+                print('CRITICAL SOURCE_EXT environment variable JSON probably malformed')
+                print('CRITICAL Correct : \'["zip","tar.gz"]\' ')
+                print('CRITICAL Not Correct: "[\'zip\',\'tar.gz\']" ')
+                exit(1)
+        else: self.source_ext = None
+
+    def release(self):
+        if self.source_ext is None:
+            self.publish_release()
+        else:
+            self.publish_prerelease()
+
+    def publish_prerelease(self):
+        '''
+        Main job to publish a pre-release.
+        '''
+        releaseNote = ReleaseNote()
+        current_message = releaseNote.get_message()
+        artifacts_list = []
+
+        # Get releases
+        artifacts_list += self._get_bin_artifacts()
+        artifacts_list.sort()
+        artifacts_list += list(map(lambda e: SourceArtifact(e), self.source_ext))
+
+        # Send result
+        note = self.template.render('notebody', {
+            'current_message': current_message,
+            'artifacts': list(map(lambda a: a.to_dict(), artifacts_list))
+        })
+        title_line = self.template.render('prerelease', {
+            'tag': os.environ['CI_COMMIT_TAG'],
+            'pipeline': os.environ['CI_PIPELINE_ID']
+        })
+        releaseNote.send_note(title_line + note)
+
+        print('Pre-release published')
+
+    def publish_release(self):
+        '''
+        Main job to publish the final release.
+        '''
+        # Change release note
+        releaseNote = ReleaseNote()
+        note = releaseNote.get_note_body()
+        title_line = self.template.render('release', {
+            'tag': os.environ['CI_COMMIT_TAG'],
+            'pipeline': os.environ['CI_PIPELINE_ID']
+        })
+        releaseNote.send_note(title_line + note)
+
+        # Update Wiki release page
+        wiki_page = ReleaseWikiPage(self.template)
+        wiki_page.add_release(os.environ['CI_COMMIT_TAG'], PlaceHolder.clear_all(note))
+        wiki_page.save()
+
+        # Keep artifacts
+        jobs = []
+        for artifact in self._get_bin_artifacts():
+            if not artifact.job in jobs:
+                jobs.append(artifact.job)
+        jobs = map(lambda j: Pipeline().find_job_id(j), jobs)
+        for job_id in jobs: Job(job_id).keep_artifacts()
+
+        print('Release published')
+
+    def _get_bin_artifacts(self):
+        '''
+        Get the binary artifacts for the current tag.
+        :return: The list of binary artifacts, based on found descriptions.
+        :rtype: list of BinArtifact
+        '''
+        DESC_EXT = '.desc'
+        artifacts = glob.glob('{}*{}'.format(self.release_bin_dir, DESC_EXT))
+        artifacts = map(lambda d: BinArtifact(self.release_bin_dir, d, DESC_EXT), artifacts)
+        artifacts = filter(lambda a: a.tag == os.environ['CI_COMMIT_TAG'], artifacts)
+        return list(artifacts)

.gitlab/releaser/releasewikipage.py

+import json
+import os
+import urllib.request
+
+from placeholder import PlaceHolder
+from projectapi import ProjectApi
+
+class ReleaseWikiPage(ProjectApi):
+    '''
+    Release Wiki page API.
+    '''
+    __PH_TAG = PlaceHolder('tag')
+    __PH_NOTE = PlaceHolder('note')
+    __PH_PREVIOUS = PlaceHolder('previous-beg')
+
+    def __init__(self, template):
+        '''
+        :param template: The template to use.
+        :type template: Template
+        '''
+        if not 'WIKI_RELEASE' in os.environ:
+            print('CRITICAL WIKI_RELEASE variable is not defined')
+            exit(1)
+        ProjectApi.__init__(self, '/wikis/{}'.format(os.environ['WIKI_RELEASE']))
+        self.template = template
+
+        # Query existing page
+        request = self.build_request()
+        response = urllib.request.urlopen(request)
+        response_data = response.read().decode()
+        data = json.loads(response_data)
+        self.page_content = data['content']
+
+    def add_release(self, tag, note):
+        '''
+        Add the release to the Wiki page.
+        '''
+        prev_tag = ReleaseWikiPage.__PH_TAG.get_content(self.page_content)
+        prev_note = ReleaseWikiPage.__PH_NOTE.get_content(self.page_content)
+        self.page_content = ReleaseWikiPage.__PH_TAG.replace_content(self.page_content, tag)
+        self.page_content = ReleaseWikiPage.__PH_NOTE.replace_content(self.page_content, note)
+        previous = self.template.render('previouswiki', {
+            'tag': prev_tag,
+            'body': prev_note
+        })
+        self.page_content = ReleaseWikiPage.__PH_PREVIOUS.insert_after(
+            self.page_content, previous)
+
+    def save(self):
+        send_data = {
+            'content': self.page_content,
+            'format': 'markdown',
+            'slug': os.environ['WIKI_RELEASE'],
+            'title': os.environ['WIKI_RELEASE']
+        }
+        send_data_serialized = json.dumps(send_data).encode('utf-8')
+        request = self.build_request(data=send_data_serialized, method='PUT')
+        request.add_header('Content-Type', 'application/json')
+        urllib.request.urlopen(request)

.gitlab/releaser/sourceartifact.py

+import os
+
+from artifact import Artifact
+from fsitemsize import FSItemSize
+
+class SourceArtifact(Artifact):
+    '''
+    A source artifact.
+    '''
+
+    def __init__(self, extention):
+        '''
+        :param extention: The extention of the source archive.
+        :type extention: str
+        '''
+        Artifact.__init__(self, 'archive.{}'.format(extention), 'Source code ({})'.format(extention), '', '', 'compression')
+
+    def _get_size(self):
+        return FSItemSize()
+
+    def _build_url(self):
+        return '{}/repository/{}/{}'.format(
+            os.environ['CI_PROJECT_URL'], os.environ['CI_COMMIT_TAG'], self.file_name)

.gitlab/releaser/template.py

+import jinja2
+import os
+
+class Template:
+    '''
+    Manages the template file. The template file is split into blocks.
+    '''
+    def __init__(self, fname):
+        '''
+        :param fname: The name of the template file.
+        :type fname: str
+        '''
+        path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+        environment = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(path),
+            trim_blocks=True
+        )
+        self.template = environment.get_template(fname)
+
+    def render(self, block, params):
+        '''
+        Render a block from the template file.
+        :param block: The name of the block to render.
+        :param params: The parameters to be used in the block.
+        :type block: str
+        :type params: dict
+        :return: The rendered block.
+        :rtype: str
+        '''
+        context = self.template.new_context(params)
+        return jinja2.utils.concat(self.template.blocks[block](context))

release/arch/linux/build-lin.sh

@@ -11,6 +11,105 @@ else
 	exit 1
 fi
 
+# ---------
+# Functions
+# ---------
+
+# Copy nw.js compiled module released library to node libraries.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy() {
+	[[ -z ${1} ]] && exit 1
+	cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
+		lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
+}
+
+# Copy nw.js compiled module library to node libraries, prefixing with node_.
+# -
+# Parameters:
+# 1. Module name.
+nw_copy_node() {
+	[[ -z ${1} ]] && exit 1
+	cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
+		lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
+}
+
+# Compile the module with nw.js.
+# -
+# Parameters:
+# 1. Module name.
+# 2. Action to be done to module after compilation, if needed.
+nw_compile() {
+	[[ -z ${1} ]] && exit 1
+	cd ${1} || exit 1
+	node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
+	node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
+	[[ -z ${2} ]] || ${2} ${1}
+	cd ..
+}
+
+# Create description.
+# -
+# Parameters:
+# 1. Initial file name.
+# 2. Building type (either “desktop” or “server”).
+# 3. Category (OS, distribution).
+create_desc() {
+	cat >"${1}".desc <<-EOF
+	{
+	  "version": "${DUNITER_TAG}",
+	  "job": "${CI_JOB_NAME}",
+	  "type": "${2^}",
+	  "category": "${3}",
+	  "arch": "x64"
+	}
+	EOF
+}
+
+# Desktop specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_desktop() {
+	cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
+}
+
+# Server specific building phase.
+# -
+# Parameters:
+# 1. Building directory.
+build_extra_server() {
+	mkdir -p "${1}/lib/systemd/system" || exit 1
+	cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
+}
+
+# Debian package building.
+# -
+# Parameters:
+# 1. Building type (either “desktop” or “server”).
+# 2. Debian package name.
+build_deb_pack() {
+	rm -rf "${RELEASES}/duniter-x64"
+	mkdir "${RELEASES}/duniter-x64" || exit 1
+	cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
+	build_extra_${1} "${RELEASES}/duniter-x64"
+	mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
+	chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
+	chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
+	sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+	cd "${RELEASES}/${1}_/"
+	zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
+
+	sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
+
+	cd "${RELEASES}"
+	fakeroot dpkg-deb --build duniter-x64 || exit 1
+	mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
+	create_desc "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" "${1}" "Linux (Ubuntu/Debian)"
+}
+
 # -----------
 # Prepare
 # -----------
@@ -43,7 +142,7 @@ RELEASES="${WORK}/releases"
 BIN="${WORK}/bin"
 
 mkdir -p "${DOWNLOADS}" "${RELEASES}" "${BIN}" || exit 1
-rm -rf "${BIN}/"*.{deb,tar.gz} # Clean up
+rm -rf "${BIN}/"*.{deb,tar.gz}{,.desc} # Clean up
 
 # -----------
 # Downloads
@@ -83,27 +182,6 @@ cp -r "${RELEASES}/duniter" "${RELEASES}/server_" || exit 1
 # Build Desktop version against nw.js
 # -------------------------------------
 
-nw_copy() {
-	[[ -z ${1} ]] && exit 1
-	cp lib/binding/Release/node-webkit-v${NW_VERSION}-linux-x64/${1}.node \
-		lib/binding/Release/node-v${ADDON_VERSION}-linux-x64/${1}.node || exit 1
-}
-
-nw_copy_node() {
-	[[ -z ${1} ]] && exit 1
-	cp lib/binding/node-webkit-v${NW_VERSION}-linux-x64/node_${1}.node \
-		lib/binding/node-v${ADDON_VERSION}-linux-x64/node_${1}.node || exit 1
-}
-
-nw_compile() {
-	[[ -z ${1} ]] && exit 1
-	cd ${1} || exit 1
-	node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} configure || exit 1
-	node-pre-gyp --runtime=node-webkit --target=${NW_VERSION} build || exit 1
-	[[ -z ${2} ]] || ${2} ${1}
-	cd ..
-}
-
 echo "${NW_RELEASE}"
 
 # FIX: bug of nw.js, we need to patch first.
@@ -162,46 +240,11 @@ cp -r "${DOWNLOADS}/node-${NVER}-linux-x64" "${RELEASES}/server_/node" || exit 1
 
 cd "${RELEASES}/desktop_"
 tar czf "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" * || exit 1
+create_desc "${BIN}/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz" "Desktop" "Linux (generic)"
 
 # -----------------------
 # Build Debian packages
 # -----------------------
 
-# Parameters
-# 1: Building directory.
-build_extra_desktop() {
-	cp -r "${ROOT}/release/extra/desktop/"* "${1}" || exit 1
-}
-
-# Parameters
-# 1: Building directory.
-build_extra_server() {
-	mkdir -p "${1}/lib/systemd/system" || exit 1
-	cp "${ROOT}/release/extra/systemd/duniter.service" "${1}/lib/systemd/system" || exit 1
-}
-
-# Parameters
-# 1: either "server" or "desktop".
-# 2: package name for Debian.
-build_deb_pack() {
-	rm -rf "${RELEASES}/duniter-x64"
-	mkdir "${RELEASES}/duniter-x64" || exit 1
-	cp -r "${ROOT}/release/extra/debian/package/"* "${RELEASES}/duniter-x64" || exit 1
-	build_extra_${1} "${RELEASES}/duniter-x64"
-	mkdir -p "${RELEASES}/duniter-x64/opt/duniter/" || exit 1
-	chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"post* || exit 1
-	chmod 755 "${RELEASES}/duniter-x64/DEBIAN/"pre* || exit 1
-	sed -i "s/Version:.*/Version:${DUNITER_DEB_VER}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
-	cd "${RELEASES}/${1}_/"
-	zip -qr "${RELEASES}/duniter-x64/opt/duniter/duniter.zip" * || exit 1
-
-	sed -i "s/Package: .*/Package: ${2}/g" "${RELEASES}/duniter-x64/DEBIAN/control" || exit 1
-
-	cd "${RELEASES}"
-	fakeroot dpkg-deb --build duniter-x64 || exit 1
-	mv duniter-x64.deb "${BIN}/duniter-${1}-${DUNITER_TAG}-linux-x64.deb" || exit 1
-}
-
 build_deb_pack desktop duniter-desktop
 build_deb_pack server duniter