Mise à jour de GitLab prévue ce samedi 23 octobre 2021 à partir de 9h00 CET

Commit f60282a5 authored by Cédric Moreau's avatar Cédric Moreau
Browse files

Merge branch 'dev'

parents 80b604b6 cfc6c9d7
......@@ -18,6 +18,9 @@ app/lib/rules/*.js
app/lib/system/*.js
app/lib/streams/*.js
app/lib/helpers/*.js
app/modules/ws2p/*.js
app/modules/ws2p/lib/*.js
app/modules/ws2p/lib/*/*.js
app/lib/*.js
app/modules/wizard.js
app/modules/router.js
......@@ -40,5 +43,7 @@ app/modules/bma/lib/entity/*.js
app/modules/bma/lib/controllers/*.js
app/modules/crawler/*.js
app/modules/crawler/lib/*.js
app/ProcessCpuProfiler.js
app/lib/common/package.js
test/*.js
test/**/*.js
\ No newline at end of file
......@@ -18,7 +18,11 @@ gui/nw
vagrant/*.log
vagrant/duniter
# Python compiled
*.pyc
# Releases
/work
*.deb
*.tar.gz
*.log
......@@ -42,13 +46,37 @@ test/blockchain/lib/*.d.ts
/server.d.ts
app/**/*.js*
app/**/*.d.ts
test/integration/revoked_pubkey_replay.js
test/integration/server-shutdown.js
test/integration/transactions-csv-cltv-sig.js
test/integration/ws2p*js
test/integration/*.js.map
test/integration/*.d.ts
test/integration/membership_chainability.js*
test/integration/membership_chainability.d.ts
test/integration/tools/toolbox.js*
test/integration/tools/toolbox.d.ts
test/integration/tools/TestUser.js*
test/integration/tools/TestUser.d.ts
test/integration/documents-currency.js*
test/integration/documents-currency.d.ts
test/integration/forwarding.js
test/integration/branches_switch.js
test/integration/branches2.js
test/integration/transactions-chaining.js
test/fast/modules/crawler/block_pulling.js*
test/fast/modules/crawler/block_pulling.d.ts
test/fast/fork*.js*
test/fast/fork*.d.ts
test/fast/proxies*.js*
test/fast/proxies*.d.ts
test/fast/modules/ws2p/*.js*
test/fast/modules/ws2p/*.d.ts
test/fast/modules/common/grammar.js*
test/fast/modules/common/grammar.d.ts
test/fast/prover/pow-1-cluster.d.ts
test/fast/prover/pow-1-cluster.js
test/fast/prover/pow-1-cluster.js.map
test/fast/protocol-local-rule-chained-tx-depth.js
test/fast/protocol-local-rule-chained-tx-depth.js.map
test/fast/protocol-local-rule-chained-tx-depth.d.ts
stages:
- github-sync
- test
- github-sync
- build
- test
- package
- prerelease
- release
push_to_github:
stage: github-sync
variables:
GIT_STRATEGY: none
tags:
- github
script:
- rm -rf ./*
- rm -rf .git
- git clone --mirror $CI_REPOSITORY_URL .
- git remote add github $GITHUB_URL_AND_KEY
- git config --global user.email "contact@duniter.org"
- git config --global user.name "Duniter"
# Job would fail if we don't remove refs about pull requests
- bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
- mv packed-refs-new packed-refs
- bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
enforce_readme:
stage: github-sync
variables:
GIT_STRATEGY: none
tags:
- github
script:
- rm -rf ./*
- rm -rf .git
- git clone $GITHUB_URL_AND_KEY .
- git config --global user.email "contact@duniter.org"
- git config --global user.name "Duniter"
- git checkout master
- cat .github/github_disclaimer.md > README.md.new
- cat README.md >> README.md.new
- mv README.md.new README.md
- git commit -am "Enforce github readme"
- git push origin master
stage: github-sync
variables:
GIT_STRATEGY: none
tags:
- redshift
script:
- rm -rf ./*
- rm -rf .git
- git clone --mirror $CI_REPOSITORY_URL .
- git remote add github $GITHUB_URL_AND_KEY
- git config --global user.email "contact@duniter.org"
- git config --global user.name "Duniter"
# Job would fail if we don't remove refs about pull requests
- bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
- mv packed-refs-new packed-refs
- bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
only:
- nodes/typescript/duniter
.nvm_env: &nvm_env
tags:
- redshift
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
build:
<<: *nvm_env
stage: build
script:
- yarn
test:
stage: test
tags:
- nodejs
image: registry.duniter.org/docker/ubuntu-node:17.10-DUNITER-2
script:
- bash -c ". ~/.nvm/nvm.sh && npm install -g yarn"
- bash -c ". ~/.nvm/nvm.sh && yarn install"
- bash -c ". ~/.nvm/nvm.sh && yarn run test-travis"
cache:
paths:
- node_modules/
<<: *nvm_env
stage: test
script:
- yarn
- yarn test
- sed -n 23p coverage/index.html
.build_releases: &build_releases
stage: package
allow_failure: false
image: duniter/release-builder:v1.0.1
tags:
- redshift-duniter-builder
when: manual
artifacts:
paths: &releases_artifacts
- work/bin/
releases:test:
<<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
artifacts:
paths: *releases_artifacts
expire_in: 4h
except:
- tags
releases:x64:
<<: *build_releases
script:
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
artifacts:
paths: *releases_artifacts
expire_in: 2 weeks
only:
- tags
.release_jobs: &release_jobs
image: tensorflow/tensorflow:latest-py3
tags:
- redshift-duniter-builder
script:
- python3 .gitlab/releaser
only:
- tags
prerelease:
<<: *release_jobs
stage: prerelease
variables:
RELEASE_BIN_DIR: work/bin/
SOURCE_EXT: '["tar.gz", "zip"]'
publish:
<<: *release_jobs
stage: release
variables:
RELEASE_BIN_DIR: work/bin/
WIKI_RELEASE: Releases
allow_failure: false
when: manual
{% block prerelease %}
# :gift: Pre-release
[Go to Pipeline page :arrow_forward:](https://git.duniter.org/nodes/typescript/duniter/pipelines/{{pipeline}})
{% endblock %}
{% block release %}
# :white_check_mark: Release
{% endblock %}
{% block notebody %}
<placeholder content="end-title" />
<placeholder content="note">
{{current_message}}
</placeholder>
## Downloads
| Category | Arch | Type | Size | File |
|----------|------|------|------|------|
{% for artifact in artifacts %}
| {{artifact.category}} | {{artifact.arch}} | {{artifact.type}} | {{artifact.size}} | [{{artifact.icon}} {{artifact.name}}]({{artifact.url}}) |
{% endfor %}
{% endblock %}
{% block previouswiki %}
## {{tag}}
{{body}}
{% endblock %}
'''
This module is meant add release notes in gitlab for the current project.
Expects to find in environment following variables:
- CI_PROJECT_URL - Automatically set by gitlab-ci
- CI_PROJECT_ID - Automatically set by gitlab-ci
- CI_COMMIT_TAG - Automatically set by gitlab-ci
- CI_PIPELINE_ID - Automatically set by gitlab-ci
- RELEASE_BIN_DIR - Directory where releases are to be found
- SOURCE_EXT - Source extensions (pre-release only)
- WIKI_RELEASE - Wiki page where releases are stored (release only)
- RELEASER_TOKEN - Token used by technical user
'''
from releaser import Releaser
Releaser().release()
class Artifact:
'''
An artifact to be uploaded.
'''
def __init__(self, file_name, category, arch, dtype, icon):
'''
:param file_name: The name of the artifact file (may have directory).
:param category: The category (OS, distrib) for the artifact.
:param arch: The architecture name.
:param dtype: The delivery type (either server or desktop).
:param icon: The name of the icon to be used for artifact representation.
:type file_name: str
:type category: str
:type arch: str
:type dtype: str
:type icon: str
'''
self.file_name = file_name
self.category = category
self.arch = arch
self.dtype = dtype
self.icon = icon
def __lt__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category < other.category or \
(self.category == other.category and self.arch < other.arch) or \
(self.category == other.category and self.arch == other.arch and self.dtype < other.dtype)
def __le__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category <= other.category or \
(self.category == other.category and self.arch <= other.arch) or \
(self.category == other.category and self.arch == other.arch and self.dtype <= other.dtype)
def __eq__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category == other.category and self.arch == other.arch and self.dtype == other.dtype
def __ne__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category != other.category or self.arch != other.arch or self.dtype != other.dtype
def __gt__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category > other.category or \
(self.category == other.category and self.arch > other.arch) or \
(self.category == other.category and self.arch == other.arch and self.dtype > other.dtype)
def __ge__(self, other):
if not isinstance(other, Artifact): raise TypeError()
return self.category >= other.category or \
(self.category == other.category and self.arch >= other.arch) or \
(self.category == other.category and self.arch == other.arch and self.dtype >= other.dtype)
def to_dict(self):
'''
:return: A dictionnary containing artifact data.
:rtype: dict
'''
return {
'name': self.file_name.split('/')[-1],
'category': self.category,
'arch': self.arch,
'type': self.dtype,
'url': self._build_url(),
'size': self._get_size(),
'icon': ':{}:'.format(self.icon)
}
def _get_size(self):
'''
:return: The size of the artifact.
:rtype: FSItemSize
'''
raise NotImplementedError()
def _build_url(self):
'''
:return: The URL which can be used to get this artifact.
:rtype: str
'''
raise NotImplementedError()
import json
import os
from artifact import Artifact
from fsitemsize import FSItemSize
class BinArtifact(Artifact):
'''
A binary artifact.
'''
def __init__(self, folder, desc_file, desc_ext):
'''
:param folder: The folder where files can be found.
:param desc_file: The name of the description file.
:param desc_ext: The extention of the description file.
:type folder: str
:type desc_file: str
:type desc_ext: str
'''
try:
description = json.load(open(desc_file))
except json.decoder.JSONDecodeError:
print('CRITICAL Description file {} could not be read'.format(desc_file))
exit(1)
self.tag = description['version']
self.job = description['job']
file_name = desc_file[:-len(desc_ext)]
Artifact.__init__(self, file_name, description['category'], description['arch'], description['type'], 'package')
def _get_size(self):
return FSItemSize(int(os.path.getsize(self.file_name)))
def _build_url(self):
return '{}/-/jobs/{}/artifacts/raw/{}'.format(
os.environ['CI_PROJECT_URL'], self.job, self.file_name)
import math
class FSItemSize:
'''
The size of a file system item.
'''
def __init__(self, bsize = None):
'''
:param bsize: Size of item in bytes.
:type bsize: int
'''
self.bsize = bsize
def __str__(self):
'''
:return: Human readable size.
:rtype: str
'''
if self.bsize is None:
return '(unknown)'
elif self.bsize == 0:
return '0 B'
size_name = ('B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB')
i = int(math.floor(math.log(self.bsize, 1024)))
power = math.pow(1024, i)
size = round(self.bsize / power, 2)
return '{} {}'.format(size, size_name[i])
import urllib.request
from projectapi import ProjectApi
class Job(ProjectApi):
'''
Job data API.
'''
def __init__(self, job_id):
'''
:param job_id: The job id.
:type job_id: int
'''
ProjectApi.__init__(self, '/jobs/{}'.format(job_id))
def keep_artifacts(self):
'''
Force artifacts to be kept forever.
'''
request = self.build_request('/artifacts/keep', method='POST')
urllib.request.urlopen(request)
class PlaceHolder:
'''
Placeholder tags in Markdown texts.
'''
__PLACEHOLDER_PART = '<placeholder'
__PLACEHOLDER_START = '<placeholder content="{}">'
__PLACEHOLDER_STOP = '</placeholder>'
__PLACEHOLDER_FULL = '<placeholder content="{}" />'
def __init__(self, content_id):
'''
:param content_id: The identifier to be used for placeholder content.
:type content_id: str
'''
self.ph_start = PlaceHolder.__PLACEHOLDER_START.format(content_id)
self.ph_stop = PlaceHolder.__PLACEHOLDER_STOP
self.ph_full = PlaceHolder.__PLACEHOLDER_FULL.format(content_id)
def get_content(self, text):
'''
:param text: The text in which to extract content.
:type text: str
:return: The content between placeholder markers.
:rtype: str
'''
pos = text.find(self.ph_start)
if pos >= 0:
text = text[pos + len(self.ph_start):]
pos = text.find(self.ph_stop)
if pos >= 0: text = text[:pos]
return text
def get_before(self, text, keep_mark=False):
'''
:param text: The text in which to extract content.
:param keep_mark: If true, the mark is kept in final text.
:type text: str
:type keep_mark: bool
:return: The content before (full) placeholder marker.
:rtype: str
'''
pos = text.find(self.ph_full)
if pos >= 0:
if keep_mark: pos += len(self.ph_full)
text = text[:pos]
return text
def get_after(self, text, keep_mark=False):
'''
:param text: The text in which to extract content.
:param keep_mark: If true, the mark is kept in final text.
:type text: str
:type keep_mark: bool
:return: The content after (full) placeholder marker.
:rtype: str
'''
pos = text.find(self.ph_full)
if pos >= 0:
if not keep_mark: pos += len(self.ph_full)
text = text[pos:]
return text
def replace_content(self, text, content):
'''
:param text: The text in which to extract content.
:param content: The new content to insert.
:type text: str
:type content: str
:return: The text where content has been replaced.
:rtype: str
'''
pos = text.find(self.ph_start)
if pos >= 0:
pos += len(self.ph_start)
text_before = text[:pos]
else:
pos = 0
text_before = ''
pos = text.find(self.ph_stop, pos)
if pos >= 0:
text_after = text[pos:]
else:
text_after = ''
return text_before + content + text_after
def insert_after(self, text, content):
'''
:param text: The text in which to extract content.
:param content: The new content to insert.
:type text: str
:type content: str
:return: The text where content has been inserted.
:rtype: str
'''
pos = text.find(self.ph_full)
if pos >= 0: pos += len(self.ph_full)
else: pos = 0
text_before = text[:pos]
text_after = text[pos:]
return text_before + content + text_after
def clear_all(text):
'''
Clear all placeholders from given text.
:param text: The text to clear.
:type text: str
:return: The clean text.
:rtype: str
'''
while True:
pos = text.find(PlaceHolder.__PLACEHOLDER_PART)
if pos < 0: break
end = text.find('>')
if end < 0: end = len(text)
text = text[:pos] + text[end + 1:]
while True:
pos = text.find(PlaceHolder.__PLACEHOLDER_STOP)
if pos < 0: break
text = text[:pos] + text[pos + len(PlaceHolder.__PLACEHOLDER_STOP):]
return text
import os
import urllib.request
class ProjectApi:
'''
Gitlab API project access.
'''
__PROJECT_URL = 'https://git.duniter.org/api/v4/projects/{}'
def __init__(self, url=''):
'''
:param url: The URL portion to add to base project URL (if needed).
:type url: str
'''
self.base_url = ProjectApi.__PROJECT_URL.format(os.environ['CI_PROJECT_ID'])
self.base_url += url
self.token = ('Private-Token', os.environ['RELEASER_TOKEN'])
def build_request(self, url='', **params):
'''
Create the request to send to project API.
:param url: The portion of URL to add to base URL (if needed).
:param params: The optional parameters.
:type url: str
:type params: dict
:return: The request, ready to be used.
:rtype: urllib.request.Request
'''
request = urllib.request.Request(self.base_url + url, **params)
request.add_header(*self.token)
return request
import json
import os
import urllib.request
from placeholder import PlaceHolder
from projectapi import ProjectApi
class ReleaseNote(ProjectApi):
'''
Release note API.
'''
__PH_TITLE = PlaceHolder('end-title')