Commit 02fe3188 authored by Éloïs's avatar Éloïs
Browse files

Merge branch 'DockerBuild' into '1.6'

Change Vagrant with docker for building Debian

See merge request nodes/typescript/duniter!1227
parents 7f9e2b52 6fc858b2
......@@ -19,6 +19,7 @@ vagrant/*.log
vagrant/duniter
# Releases
/work
*.deb
*.tar.gz
*.log
......
stages:
- github-sync
- test
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
- github-sync
- build
- test
- releases
- releases-page
push_to_github:
stage: github-sync
variables:
GIT_STRATEGY: none
tags:
- redshift
before_script:
- ''
script:
- rm -rf ./*
- rm -rf .git
......@@ -25,32 +21,77 @@ push_to_github:
- bash -c "cat packed-refs | grep -v 'refs/pull' > packed-refs-new; echo 'Removed pull refs.'"
- mv packed-refs-new packed-refs
- bash -c "git push --force --mirror github 2>&1 | grep -v duniter-gitlab; echo $?"
enforce_readme:
stage: github-sync
variables:
GIT_STRATEGY: none
tags:
- redshift
before_script:
- ''
script:
- rm -rf ./*
- rm -rf .git
- git clone $GITHUB_URL_AND_KEY .
- git config --global user.email "contact@duniter.org"
- git config --global user.name "Duniter"
- git checkout master
- cat .github/github_disclaimer.md > README.md.new
- cat README.md >> README.md.new
- mv README.md.new README.md
- git commit -am "Enforce github readme"
- git push origin master
build:
stage: build
tags:
- redshift
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
script:
- yarn
test:
stage: test
tags:
- redshift
before_script:
- export NVM_DIR="$HOME/.nvm"
- . "$NVM_DIR/nvm.sh"
script:
- yarn
- yarn test
releases:test:
stage: releases
image: duniter/release-builder:v1.0.1
tags:
- redshift-duniter-builder
variables:
DAY: $(date +%Y%m%d)
HOUR: $(date +%H%M)
SEC: $(date +%S)
script:
- bash "release/arch/linux/build-lin.sh" "$(date +%Y%m%d).$(date +%H%M).$(date +%S)"
artifacts:
paths:
- work/bin/
expire_in: 8h
when: manual
except:
- tags
releases:
stage: releases
image: duniter/release-builder:v1.0.1
tags:
- redshift-duniter-builder
script:
- bash "release/arch/linux/build-lin.sh" "${CI_COMMIT_TAG#v}"
artifacts:
paths:
- work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb
- work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz
- work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb
expire_in: 8h
when: manual
only:
- tags
- master
releases-message:
stage: releases-page
image: tensorflow/tensorflow:latest-py3
tags:
- redshift-duniter-builder
variables:
JOB_ARTIFACTS: 'releases'
EXPECTED_ARTIFACTS: '["work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.deb","work/bin/duniter-desktop-${CI_COMMIT_TAG}-linux-x64.tar.gz","work/bin/duniter-server-${CI_COMMIT_TAG}-linux-x64.deb"]'
script:
- python3 .gitlab/releaser.py
when: manual
only:
- tags
- master
{{current_message}}
# Downloads
{% for artifact in artifacts %}
***
[{{artifact.icon}} {{artifact.name}}]({{artifact.url}})
_{{artifact.size}}_
***
{% endfor %}
#!/usr/bin/python3
'''
This module is meant to overload the release note in gitlab for the current project.
Expects to find in environment following variables:
- CI_PROJECT_URL - Automatically set by gitlab-ci
- CI_COMMIT_TAG - Automatically set by gitlab-ci
- CI_PROJECT_ID - Automatically set by gitlab-ci
- CI_COMMIT_TAG - Automatically set by gitlab-ci
- RELEASER_TOKEN - Token used by technical user
- JOB_ARTIFACTS - String containing job name containing all artifacts, to set manually
- EXPECTED_ARTIFACTS - List containing all artifacts generated to set manually
'''
import math
import urllib.request
import urllib.error
import json
import os
import jinja2
def convert_size(size_bytes):
'''Print proper size'''
if size_bytes == 0:
return '0B'
size_name = ('B', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB')
i = int(math.floor(math.log(size_bytes, 1024)))
power = math.pow(1024, i)
size = round(size_bytes / power, 2)
return '%s %s' % (size, size_name[i])
def get_current_message():
'''Get current release message'''
ci_project_id = os.environ['CI_PROJECT_ID']
ci_commit_tag = os.environ['CI_COMMIT_TAG']
tag_url = 'https://git.duniter.org/api/v4/projects/'
tag_url += ci_project_id
tag_url += '/repository/tags/'
tag_url += ci_commit_tag
request = urllib.request.Request(tag_url)
response = urllib.request.urlopen(request)
response_data = response.read().decode()
data = json.loads(response_data)
if data['release'] is None:
return False, ''
else:
return True, data['release']['description'].split('# Downloads')[0]
def build_artifact_url(artifact, source):
'''Given an artifact name, builds the url to download it'''
job_artifacts = os.environ['JOB_ARTIFACTS']
ci_project_url = os.environ['CI_PROJECT_URL']
ci_commit_tag = os.environ['CI_COMMIT_TAG']
if source:
source_url = ci_project_url
source_url += '/repository/'
source_url += ci_commit_tag
source_url += '/archive.'
source_url += artifact
return source_url
else:
artifact_url = ci_project_url
artifact_url += '/-/jobs/artifacts/'
artifact_url += ci_commit_tag
artifact_url += '/raw/'
artifact_url += artifact
artifact_url += '?job='
artifact_url += job_artifacts
return artifact_url
def get_artifact_weight(location):
'''Retrieve size of artifacts'''
size = os.path.getsize(location)
return convert_size(int(size))
def build_compiled_message(current_message):
'''Create a new release message using the release template'''
expected_artifacts = os.environ['EXPECTED_ARTIFACTS']
try:
expected_artifacts = json.loads(expected_artifacts)
except json.decoder.JSONDecodeError:
print('CRITICAL EXPECTED_ARTIFACTS environment variable JSON probably malformed')
print('CRITICAL Correct : \'["test_linux.txt","test_windows.txt"]\' ')
print('CRITICAL Not Correct: "[\'test_linux.txt\',\'test_windows.txt\']" ')
exit(1)
artifacts_list = []
for artifact in expected_artifacts:
artifact_dict = {
'name': artifact.split('/')[-1],
'url': build_artifact_url(artifact, False),
'size': get_artifact_weight(artifact),
'icon': ':package:'
}
artifacts_list.append(artifact_dict)
j2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(
os.path.dirname(os.path.abspath(__file__))
),
trim_blocks=True
)
# pylint: disable=maybe-no-member
template = j2_env.get_template('release_template.md')
return template.render(
current_message=current_message,
artifacts=artifacts_list
)
def send_compiled_message(exists_release, compiled_message):
'''Send to gitlab new message'''
releaser_token = os.environ['RELEASER_TOKEN']
ci_project_id = os.environ['CI_PROJECT_ID']
ci_commit_tag = os.environ['CI_COMMIT_TAG']
release_url = 'https://git.duniter.org/api/v4/projects/'
release_url += ci_project_id
release_url += '/repository/tags/'
release_url += ci_commit_tag
release_url += '/release'
if exists_release:
# We need to send a PUT request
method = 'PUT'
else:
# We need to send a POST request
method = 'POST'
send_data = {
'tag_name':ci_commit_tag,
'description':compiled_message
}
send_data_serialized = json.dumps(send_data).encode('utf-8')
request = urllib.request.Request(release_url, data=send_data_serialized, method=method)
request.add_header('Private-Token', releaser_token)
request.add_header('Content-Type', 'application/json')
response = urllib.request.urlopen(request)
def main():
'''Execute main scenario'''
exists_release, current_message = get_current_message()
compiled_message = build_compiled_message(current_message)
send_compiled_message(exists_release, compiled_message)
print('Artifacts uploaded successfully')
main()
......@@ -86,7 +86,7 @@ mkdir -p duniter_release
cp -R ${SRC}/* duniter_release/
# Creating DEB packaging
mv duniter_release/release/arch/debian/package duniter-${ARCH}
mv duniter_release/release/extra/debian/package duniter-${ARCH}
mkdir -p duniter-${ARCH}/opt/duniter/
chmod 755 duniter-${ARCH}/DEBIAN/post*
chmod 755 duniter-${ARCH}/DEBIAN/pre*
......
# -*- mode: ruby -*-
# vi: set ft=ruby :
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure("2") do |config|
# The most common configuration options are documented and commented below.
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
# Every Vagrant development environment requires a box. You can search for
# boxes at https://atlas.hashicorp.com/search.
config.vm.box = "https://s3.eu-central-1.amazonaws.com/duniter/vagrant/duniter_trusty64.box"
config.vm.provision :shell, path: "bootstrap.sh"
# Disable automatic box update checking. If you disable this, then
# boxes will only be checked for updates when the user runs
# `vagrant box outdated`. This is not recommended.
# config.vm.box_check_update = false
# Create a forwarded port mapping which allows access to a specific port
# within the machine from a port on the host machine. In the example below,
# accessing "localhost:8080" will access port 80 on the guest machine.
# config.vm.network "forwarded_port", guest: 80, host: 8080
# Create a private network, which allows host-only access to the machine
# using a specific IP.
# config.vm.network "private_network", ip: "192.168.33.10"
# Create a public network, which generally matched to bridged network.
# Bridged networks make the machine appear as another physical device on
# your network.
# config.vm.network "public_network"
# Share an additional folder to the guest VM. The first argument is
# the path on the host to the actual folder. The second argument is
# the path on the guest to mount the folder. And the optional third
# argument is a set of non-required options.
# config.vm.synced_folder "../data", "/vagrant_data"
# Provider-specific configuration so you can fine-tune various
# backing providers for Vagrant. These expose provider-specific options.
# Example for VirtualBox:
#
config.vm.provider "virtualbox" do |vb|
# Display the VirtualBox GUI when booting the machine
#vb.gui = true
# Customize the amount of memory on the VM:
vb.memory = "2048"
end
#
# View the documentation for the provider you are using for more
# information on available options.
# Define a Vagrant Push strategy for pushing to Atlas. Other push strategies
# such as FTP and Heroku are also available. See the documentation at
# https://docs.vagrantup.com/v2/push/atlas.html for more information.
# config.push.define "atlas" do |push|
# push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME"
# end
# Enable provisioning with a shell script. Additional provisioners such as
# Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
# documentation for more information about their specific syntax and use.
# config.vm.provision "shell", inline: <<-SHELL
# apt-get update
# apt-get install -y apache2
# SHELL
end
#!/bin/bash
# Yarn
curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
# System tools
apt-get update
apt-get install --yes git curl build-essential yarn python-minimal zip
# User installation
sudo su vagrant -c "bash /vagrant/user-bootstrap.sh"
#!/bin/bash
# NVM
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
# Prepare
NODE_VERSION=8.9.1
NVER="v$NODE_VERSION"
DUNITER_TAG=$1
ADDON_VERSION=57
NW_VERSION=0.24.4
NW_RELEASE="v${NW_VERSION}"
NW="nwjs-${NW_RELEASE}-linux-x64"
NW_GZ="${NW}.tar.gz"
nvm install ${NVER}
nvm use ${NVER}
npm install -g node-pre-gyp
npm install -g nw-gyp
# Folders
ROOT=`pwd`
DOWNLOADS="$ROOT/downloads"
RELEASES="$ROOT/releases"
mkdir -p "$DOWNLOADS"
# -----------
# Clean sources + releases
# -----------
rm -rf "$DOWNLOADS/duniter"
rm -rf "$RELEASES"
rm -rf /vagrant/*.deb
rm -rf /vagrant/*.tar.gz
# -----------
# Downloads
# -----------
cd "$DOWNLOADS"
if [ ! -d "$DOWNLOADS/duniter" ]; then
mv /vagrant/duniter-source duniter
cd duniter
git checkout "v${DUNITER_TAG}"
cd ..
fi
DUNITER_DEB_VER=" $DUNITER_TAG"
DUNITER_TAG="v$DUNITER_TAG"
if [ ! -f "$DOWNLOADS/$NW_GZ" ]; then
wget https://dl.nwjs.io/${NW_RELEASE}/${NW_GZ}
tar xvzf ${NW_GZ}
fi
if [ ! -f "$DOWNLOADS/node-${NVER}-linux-x64.tar.gz" ]; then
# Download Node.js and package it with the sources
wget http://nodejs.org/dist/${NVER}/node-${NVER}-linux-x64.tar.gz
tar xzf node-${NVER}-linux-x64.tar.gz
fi
# -----------
# Releases
# -----------
rm -rf "$RELEASES"
mkdir -p "$RELEASES"
cp -r "$DOWNLOADS/duniter" "$RELEASES/duniter"
cd "$RELEASES"
# NPM build
cp -r duniter _npm
# Releases builds
cd ${RELEASES}/duniter
# Remove git files
rm -Rf .git
[[ $? -eq 0 ]] && echo ">> VM: building modules..."
[[ $? -eq 0 ]] && npm install
# Duniter UI
[[ $? -eq 0 ]] && npm install duniter-ui@1.6.x
[[ $? -eq 0 ]] && npm prune --production
cp -r "$RELEASES/duniter" "$RELEASES/desktop_"
cp -r "$RELEASES/duniter" "$RELEASES/server_"
# -------------------------------------------------
# Build Desktop version (Nw.js is embedded)
# -------------------------------------------------
cd "$RELEASES/desktop_"
echo "$NW_RELEASE"
cd "$RELEASES/desktop_/node_modules/wotb"
# FIX: bug of nw.js, we need to patch first.
# TODO: remove this patch once a correct version of Nw.js is out (NodeJS 8 or 9 if the above modules are compliant)
cp /vagrant/0.24.4_common.gypi ~/.nw-gyp/0.24.4/common.gypi
#yarn --build-from-source
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION configure
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION build
cp lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64/wotb.node lib/binding/Release/node-v$ADDON_VERSION-linux-x64/wotb.node
cd "$RELEASES/desktop_/node_modules/naclb"
#npm install --build-from-source
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION configure
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION build
cp lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64/naclb.node lib/binding/Release/node-v$ADDON_VERSION-linux-x64/naclb.node
cd "$RELEASES/desktop_/node_modules/scryptb"
#npm install --build-from-source
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION configure
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION build
cp lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64/scryptb.node lib/binding/Release/node-v$ADDON_VERSION-linux-x64/scryptb.node
cd "$RELEASES/desktop_/node_modules/sqlite3"
#npm install --build-from-source
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION configure
node-pre-gyp --runtime=node-webkit --target=$NW_VERSION build
cp lib/binding/node-webkit-$NW_RELEASE-linux-x64/node_sqlite3.node lib/binding/node-v$ADDON_VERSION-linux-x64/node_sqlite3.node
# Unused binaries
cd "$RELEASES/desktop_/"
rm -rf node_modules/sqlite3/build
#rm -rf node_modules/naclb/build
#rm -rf node_modules/wotb/build
#rm -rf node_modules/scryptb/build
## Install Nw.js
mkdir -p "$RELEASES/desktop_release"
# -------------------------------------------------
# Build Desktop version .tar.gz
# -------------------------------------------------
cp -r $DOWNLOADS/${NW}/* "$RELEASES/desktop_release/"
# Embed Node.js with Nw.js to make Duniter modules installable
cp -r ${DOWNLOADS}/node-${NVER}-linux-x64/lib "$RELEASES/desktop_release/"
cp -r ${DOWNLOADS}/node-${NVER}-linux-x64/include "$RELEASES/desktop_release/"
cp -r ${DOWNLOADS}/node-${NVER}-linux-x64/bin "$RELEASES/desktop_release/"
# Add some specific files for GUI
cp ${RELEASES}/desktop_/gui/* "$RELEASES/desktop_release/"
# Add Duniter sources
cp -R $RELEASES/desktop_/* "$RELEASES/desktop_release/"
## Insert Nw specific fields while they do not exist (1.3.3)
sed -i "s/\"main\": \"index.js\",/\"main\": \"index.html\",/" "$RELEASES/desktop_release/package.json"
# Add links for Node.js + NPM
cd "$RELEASES/desktop_release/bin"
ln -s ../lib/node_modules/npm/bin/npm-cli.js ./npm -f
cd ..
ln -s ./bin/node node -f
ln -s ./bin/npm npm -f
#sed -i "s/\"node-main\": \"\.\.\/sources\/bin\/duniter\",/\"node-main\": \".\/bin\/duniter\",/" "$RELEASES/desktop_release/package.json"
# Create a copy for TGZ binary
cp -R "$RELEASES/desktop_release" "$RELEASES/desktop_release_tgz"
#cd "$RELEASES/desktop_release_tgz/"
#rm -rf node_modules/sqlite3/lib/binding/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/wotb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/naclb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/scryptb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
cd "$RELEASES/desktop_release_tgz"
tar czf /vagrant/duniter-desktop-${DUNITER_TAG}-linux-x64.tar.gz * --exclude ".git" --exclude "coverage" --exclude "test"
# -------------------------------------------------
# Build Desktop version .deb
# -------------------------------------------------
# Create .deb tree + package it
#cp -r "$RELEASES/desktop_release/release/arch/debian/package" "$RELEASES/duniter-x64"
cp -r "/vagrant/package" "$RELEASES/duniter-x64"
mkdir -p "$RELEASES/duniter-x64/opt/duniter/"
chmod 755 ${RELEASES}/duniter-x64/DEBIAN/post*
chmod 755 ${RELEASES}/duniter-x64/DEBIAN/pre*
sed -i "s/Version:.*/Version:$DUNITER_DEB_VER/g" ${RELEASES}/duniter-x64/DEBIAN/control
cd ${RELEASES}/desktop_release/
#rm -rf node_modules/sqlite3/lib/binding/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/wotb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/naclb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/scryptb/lib/binding/Release/node-webkit-$NW_RELEASE-linux-x64
#rm -rf node_modules/sqlite3/lib/binding/node-v$ADDON_VERSION-linux-x64
#rm -rf node_modules/wotb/lib/binding/Release/node-v$ADDON_VERSION-linux-x64
#rm -rf node_modules/naclb/lib/binding/Release/node-v$ADDON_VERSION-linux-x64
#rm -rf node_modules/scryptb/lib/binding/Release/node-v$ADDON_VERSION-linux-x64
zip -qr ${RELEASES}/duniter-x64/opt/duniter/duniter-desktop.nw *
sed -i "s/Package: .*/Package: duniter-desktop/g" ${RELEASES}/duniter-x64/DEBIAN/control
cd ${RELEASES}/
fakeroot dpkg-deb --build duniter-x64
mv duniter-x64.deb /vagrant/duniter-desktop-${DUNITER_TAG}-linux-x64.deb
# -------------------------------------------------
# Build Server version (Node.js is embedded, not Nw.js)
# -------------------------------------------------
cd ${RELEASES}
rm -rf duniter-server-x64
cp -r duniter-x64 duniter-server-x64
# Remove Nw.js
rm -rf duniter-server-x64/opt/duniter/duniter-desktop.nw*
cd ${RELEASES}/server_
cp -r ${DOWNLOADS}/node-${NVER}-linux-x64 node
zip -qr ${RELEASES}/duniter-server-x64/opt/duniter/duniter-desktop.nw *
cd ${RELEASES}
sed -i "s/Package: .*/Package: duniter/g" ${RELEASES}/duniter-server-x64/DEBIAN/control
rm -rf ${RELEASES}/duniter-server-x64/usr
fakeroot dpkg-deb --build duniter-server-x64
mv duniter-server-x64.deb /vagrant/duniter-server-${DUNITER_TAG}-linux-x64.deb
#!/bin/bash
# NVM
curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.1/install.sh | bash
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm
# Node.js
nvm install 6
# node-pre-gyp
npm install -g nw-gyp node-pre-gyp
#!/bin/bash
if [[ -z "${1}" ]]; then
echo "Fatal: no version given to build script"
exit 1
fi
if [[ -s "$NVM_DIR/nvm.sh" ]]; then
source "$NVM_DIR/nvm.sh"
else