From fadea9b58c1e41d350b03644bb5d8cb0274be80b Mon Sep 17 00:00:00 2001 From: AlmAck Date: Tue, 2 Jan 2018 21:04:08 +0100 Subject: [PATCH] Enable gitlab-ci on [gtk] repository --- .build-bin/deploy.sh | 19 +++ .build-bin/import-validpgpkeys.sh | 9 ++ .build-bin/prepare.sh | 27 +++++ .build-lib/LICENSE | 37 ++++++ .build-lib/ci-deploy-library.sh | 70 +++++++++++ .build-lib/ci-library.sh | 189 ++++++++++++++++++++++++++++++ .gitlab-ci.yml | 99 ++++++++++++++++ 7 files changed, 450 insertions(+) create mode 100755 .build-bin/deploy.sh create mode 100755 .build-bin/import-validpgpkeys.sh create mode 100755 .build-bin/prepare.sh create mode 100644 .build-lib/LICENSE create mode 100644 .build-lib/ci-deploy-library.sh create mode 100644 .build-lib/ci-library.sh create mode 100644 .gitlab-ci.yml diff --git a/.build-bin/deploy.sh b/.build-bin/deploy.sh new file mode 100755 index 0000000..2f20907 --- /dev/null +++ b/.build-bin/deploy.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +SCRIPT_DIR=$( dirname $( readlink -e $0 ) ) +source "$SCRIPT_DIR/../.build-lib/ci-library.sh" +source "$SCRIPT_DIR/../.build-lib/ci-deploy-library.sh" + +# get the list of packages to upload +_do list_upload_packages + +if [ -z "$UPLOAD_LIST" ]; then + _log success "Done nothing to upload!" +else + # rsync upload + _do upload_files + + # akbm to add to the repository + _do update_remote_db + + _log success "Done uploading!" +fi diff --git a/.build-bin/import-validpgpkeys.sh b/.build-bin/import-validpgpkeys.sh new file mode 100755 index 0000000..e57dcc1 --- /dev/null +++ b/.build-bin/import-validpgpkeys.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +SCRIPT_DIR=$( dirname $( readlink -e $0 ) ) +source "$SCRIPT_DIR/../.build-lib/ci-library.sh" + +_do list_packages + + +# `gpg --recv-key` requires write access to the current user's home directory! +_do gpg --recv-key $(get_validpgpkeys) \ No newline at end of file diff --git a/.build-bin/prepare.sh b/.build-bin/prepare.sh new file mode 100755 index 0000000..34f4ced --- /dev/null +++ b/.build-bin/prepare.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +SCRIPT_DIR=$( dirname $( readlink -e $0 ) ) +source "$SCRIPT_DIR/../.build-lib/ci-library.sh" + +# prepare the build environment +_log command "Setting up locale.conf..." +_do touch "/etc/locale.conf" +_do sh -c "echo 'LANG=C' >> '/etc/locale.conf'" +_do sh -c "echo 'LC_MESSAGES=C' >> '/etc/locale.conf'" + +# create a local makepkg settings +_log command "Setting up makepkg.conf..." +_do echo "source /etc/makepkg.conf + +#-- Make Flags: change this for DistCC/SMP systems +MAKEFLAGS=\"-j$(($(nproc)+1))\" + +#-- Packager: name/email of the person or organization building packages +PACKAGER=\"$GITLAB_USER_ID <$GITLAB_USER_EMAIL>\"" > /home/builder/makepkg.conf + +# patch current makepkg with chakra custom mods +_log command "Patching makepkg..." +_do wget https://code.chakralinux.org/tools/chakrabuildsystem/raw/master/chakra/bin/makepkg +_do cp makepkg /usr/bin/makepkg + +# force to use the default chakra rsync +# sed -i '1s/^/ /' file diff --git a/.build-lib/LICENSE b/.build-lib/LICENSE new file mode 100644 index 0000000..4269674 --- /dev/null +++ b/.build-lib/LICENSE @@ -0,0 +1,37 @@ +ci-library.sh is based upon +https://github.com/Alexpux/MINGW-packages/blob/master/ci-library.sh +licensed under the "3-clause BSD" license found below. + +Modifications are licensed under the Apache License, Version 2.0 (the "License") +- see LICENSE file in the repository root. + + +"3-clause BSD" license: + +Copyright (c) 2013, Алексей +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the {organization} nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/.build-lib/ci-deploy-library.sh b/.build-lib/ci-deploy-library.sh new file mode 100644 index 0000000..992df28 --- /dev/null +++ b/.build-lib/ci-deploy-library.sh @@ -0,0 +1,70 @@ +# ci-depoly-library depends on ci-library + +UPLOAD_LIST=() + +# Execute command and stop execution if the command fails +function _do_deploy() { + CMD=$@ + _log command "$CMD" + $CMD || { _log failure "FAILED: $CMD"; _unlock_repo; exit 1; } + return $? +} + +function _do_akbm() { + local output + CMD=$@ + _log command "$CMD" + output=$($CMD) + + if ! [[ "$output" == *"::SUCCESS::"* ]]; then + _log failure "FAILED: $CMD output: $output" + _unlock_repo + exit 1 + fi + return $? +} + +# performs a remote lock over a repository +function _lock_repo() { + _do_akbm ssh $SSH_USER@$DEPLOY_SERVER -p $SSH_PORT "akbm --repo-name $DEPLOY_REPO --arch x86_64 --lock" +} + +# performs a remote unlock over a repository +function _unlock_repo() { + _do_akbm ssh $SSH_USER@$DEPLOY_SERVER -p $SSH_PORT "akbm --repo-name $DEPLOY_REPO --arch x86_64 --unlock" +} + +function _upload_files() { + local -a files=( $* ) # files to upload + + rsync -rltoDvh \ + --omit-dir-times \ + --numeric-ids \ + --progress \ + --delay-updates \ + -e "ssh -p $SSH_PORT" \ + "${files[@]}" $SSH_USER@$DEPLOY_SERVER:/srv/www/rsync.chakralinux.org/packages/$DEPLOY_REPO/x86_64/ +} + +function upload_files() { + _log build_step "Start uploading to $DEPLOY_REPO the following packages: ${UPLOAD_LIST[@]}" + _do _lock_repo + _do_deploy _upload_files "${UPLOAD_LIST[@]}" + _do _unlock_repo + _log success "rsync upload done" +} + +function update_remote_db() { + _log build_step "Start importing pkgs with akbm" + local -a file_names=( ${UPLOAD_LIST[@]##*/} ) + + _do _lock_repo + # we can no tuse _do in this case, because we have to parse the output to know if was successfully executed + _do_akbm ssh "$SSH_USER@$DEPLOY_SERVER" -p "$SSH_PORT" "akbm --repo-name $DEPLOY_REPO --arch x86_64 --repo-add" "${file_names[@]}" + _do _unlock_repo +} + +_ensure-var "DEPLOY_REPO" +_ensure-var "DEPLOY_SERVER" +_ensure-var "SSH_USER" +_ensure-var "SSH_PORT" diff --git a/.build-lib/ci-library.sh b/.build-lib/ci-library.sh new file mode 100644 index 0000000..4f3dcb2 --- /dev/null +++ b/.build-lib/ci-library.sh @@ -0,0 +1,189 @@ +# Based upon +# https://github.com/Alexpux/MINGW-packages/blob/master/ci-library.sh +# and https://github.com/episource/archlinux-overlay + + +PACKAGES=() + +# Print a colored log message +function _log() { + local type="${1}" + shift + local msg="${@}" + + local normal='\e[0m' + local red='\e[1;31m' + local green='\e[1;32m' + local yellow='\e[1;33m' + local cyan='\e[1;36m' + + case "${type}" in + failure) echo -e "$red$msg$normal" ;; + success) echo -e "$green$msg$normal" ;; + build_step) echo -e "$green$msg$normal" ;; + command) echo -e "$cyan$msg$normal" ;; + message) echo -e "$msg" ;; + esac +} + +# Execute command and stop execution if the command fails +function _do() { + CMD=$@ + _log command "$CMD" + $CMD || { _log failure "FAILED: $CMD"; exit 1; } + return $? +} + +# Ensure that the given environment variable has been defined and is not empty +function _ensure-var() { + local -n VARNAME=$1 + if [[ -z ${VARNAME+x} ]]; then + _log failure "Environment variable $1 not defined." + exit 1 + fi +} + + +# Package provides another (ignoring version constraints) +function _package_provides() { + local package="${1}" + local another_without_version="${2%%[<>=]*}" + local pkgname provides + _package_info "${package}" pkgname provides + for pkg_name in "${pkgname[@]}"; do [[ "${pkg_name}" = "${another_without_version}" ]] && return 0; done + for provided in "${provides[@]}"; do [[ "${provided}" = "${another_without_version}" ]] && return 0; done + return 1 +} + +# Get package information +function _package_info() { + local package="${1}" + local properties=("${@:2}") + for property in "${properties[@]}"; do + local -n nameref_property="${property}" + nameref_property=($( + source "${package}/PKGBUILD" + declare -n nameref_property="${property}" + echo "${nameref_property[@]}")) + done +} + +# Add package to build after required dependencies +function _build_add() { + local package="${1}" + local depends makedepends + for sorted_package in "${sorted_packages[@]}"; do + [[ "${sorted_package}" = "${package}" ]] && return 0 + done + _package_info "${package}" depends makedepends + for dependency in "${depends[@]}" "${makedepends[@]}"; do + for unsorted_package in "${PACKAGES[@]}"; do + [[ "${package}" = "${unsorted_package}" ]] && continue + _package_provides "${unsorted_package}" "${dependency}" && _build_add "${unsorted_package}" + done + done + sorted_packages+=("${package}") +} + +# Convert lines to array +_as_list() { + local -n nameref_list="${1}" + local filter="${2}" + local strip="${3}" + local lines="${4}" + local result=1 + nameref_list=() + while IFS= read -r line; do + test -z "${line}" && continue + result=0 + [[ "${line}" = ${filter} ]] && nameref_list+=("${line/${strip}/}") + done <<< "${lines}" + return "${result}" +} + +# Changes since master or from head +function _list_changes() { + local list_name="${1}" + local filter="${2}" + local strip="${3}" + local git_options=("${@:4}") + #_as_list "${list_name}" "${filter}" "${strip}" "$(git log "${git_options[@]}" master.. | sort -u)" || + #_as_list "${list_name}" "${filter}" "${strip}" "$(git log "${git_options[@]}" HEAD^.. | sort -u)" + _as_list "${list_name}" "${filter}" "${strip}" "$(git diff-tree "${git_options[@]}" HEAD)" +} + +# Added commits +function list_commits() { + _list_changes commits '*' '#*::' --pretty=format:'%ai::[%h] %s' +} + +# Changed recipes +function list_packages() { + local _packages + local _orders + _list_changes _packages '*/PKGBUILD' '%/PKGBUILD' --no-commit-id --pretty=format: --name-only -r || return 1 + for _package in "${_packages[@]}"; do + PACKAGES+=("${_package}") + done + + # check if there are some .order file + _list_changes _orders '*.order' '%' --no-commit-id --pretty=format: --name-only -r || return 1 + for _order in "${_orders[@]}"; do + exec 3<$_order + while read -u3 _line; do + [[ $_line =~ ^[:blank:]*$ ]] && continue + + local comment_re="^[:blank:]*#" + [[ $_line =~ $comment_re ]] && continue + + PACKAGES+=("${_line}") + done + done + return 0 +} + +# extracts all 'validpgpkeys' from the PKGBUILDs +# extracts all 'validpgpkeys' listed in the PKGBUILDs belonging to $PACKAGES +function get_validpgpkeys() { + _VALIDPGPKEYS=() + for p in "${PACKAGES[@]}"; do + local validpgpkeys=() + _package_info "$p" validpgpkeys + _VALIDPGPKEYS+=$validpgpkeys + done + + echo "${_VALIDPGPKEYS[@]}" +} + +# Sort packages by dependency +# reorders $PACKAGES such that dependencies are built first +function sort_packages_by_dependency() { + local sorted_packages=() + for p in "${PACKAGES[@]}"; do + _build_add "${p}" + done + PACKAGES=("${sorted_packages[@]}") +} + +# Build all packages defined in array variable PACKAGES +# builds all $PACKAGES in the given order +function build_packages() { + _log build_step "Start building packages: ${PACKAGES[@]}" + _do mkdir -p "$REPODIR" + + for p in "${PACKAGES[@]}"; do + cd $p + _log command "Building pkg: $p" + PKGEXT=".pkg.tar.xz" PKGDEST="$REPODIR" \ + _do makepkg --noconfirm --noprogressbar --nosign --syncdeps --cleanbuild + cd - > /dev/null + # resync to update local repo + #_do pacman -Sy --noconfirm + done + + _log success "Done building packages!" +} + + +_ensure-var "REPODIR" +#_ensure-var "REPONAME" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..c655f57 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,99 @@ +image: chakralinux/chakra-bootstrap:latest + +variables: + REPODIR: "$CI_PROJECT_DIR" + +stages: + - package + - sign + - deploy + + +before_script: + - pacman -Syyu --needed --noconfirm base-devel gettext wget openssh git rsync + + # The repository build script is run by user 'builder'. The build scripts needs + # to install dependencies using pacman. This requires root permissions. + # (Note: the build script itself can't be run as root, as makepkg would + # complain) + - useradd -m -G users -s /bin/bash builder + - "echo 'builder ALL=(ALL) NOPASSWD: /usr/bin/pacman' >> /etc/sudoers" + #- echo -en "builder ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/10-builder + + # SSH setup + # Run ssh-agent (inside the build environment) + - eval $(ssh-agent -s) + + # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store + - ssh-add <(echo "$SSH_PRIVATE_KEY") + + # For Docker builds disable host key checking. Be aware that by adding that + # you are suspectible to man-in-the-middle attacks. + # WARNING: Use this only with the Docker executor, if you use it with shell + # you will overwrite your user's SSH config. + #- mkdir -p ~/.ssh + #- '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + # In order to properly check the server's host key, assuming you created the + # SSH_SERVER_HOSTKEYS variable previously, uncomment the following two lines + # instead. + - mkdir -p ~/.ssh + - '[[ -f /.dockerenv ]] && echo "$SSH_SERVER_HOSTKEYS" > ~/.ssh/known_hosts' + - echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa + - chmod 600 ~/.ssh/id_rsa && chmod 700 ~/.ssh + + # GPG setup + - gpg -v --batch --import <(echo "$GPG_PRIVATE_KEY") + - echo "allow-loopback-pinentry" > ~/.gnupg/gpg-agent.conf + - gpg-connect-agent reloadagent /bye + + # setup basic environment settings, as root + - .build-bin/prepare.sh + # import pgp keys flagged as valid + - sudo -u builder -E -H .build-bin/import-validpgpkeys.sh + +build_repo: + stage: package + script: + # the build script can't be run as root, as makepkg would complain... + - sudo -u builder -E -H .build-bin/build.sh + + artifacts: + # expire artifacts per default - the gitlab web frontend can be used to keep + # artifacts of interest for an unlimited time + expire_in: 1 week + paths: + - ./*.pkg.tar.xz + name: "$CI_BUILD_NAME" + cache: + paths: + - /var/cache/pacman/pkg + tags: + - PKGBUILD + +sign_pkgs: + stage: sign + script: + - echo "$GPG_PASSWORD" | gpg -sb --pinentry-mode loopback --passphrase-fd 0 ./*.pkg.tar.xz + artifacts: + # expire artifacts per default - the gitlab web frontend can be used to keep + # artifacts of interest for an unlimited time + expire_in: 1 week + paths: + - ./*.pkg.tar.xz.sig + name: "$CI_BUILD_NAME" + tags: + - signature + +deploy_repo: + stage: deploy + variables: + DEPLOY_REPO: "$DEPLOY_REPO" + DEPLOY_SERVER: "$DEPLOY_SERVER" + SSH_USER: "$SSH_USER" + SSH_PORT: "$SSH_PORT" + environment: staging-repo + script: + - .build-bin/deploy.sh + when: manual + tags: + - rsync