diff --git a/config b/config index 13c4ed5..4354d2b 100644 --- a/config +++ b/config @@ -11,6 +11,7 @@ DEBUGPKGPOOL='' STAGING_REPOS=() TESTING_REPOS=() STABLE_REPOS=() +declare -A ACL=() # VCS backend VCS=svn @@ -48,6 +49,12 @@ LIST="arch-dev-public@lists.archlinux.org" #LIST="aaronmgriffin@gmail.com" FROM="repomaint@archlinux.org" +# curated PATH to sanitize executables in a portable way +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export PATH + +AUTHORS=/notexists + # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname "${BASH_SOURCE[0]}")/config.local"} [[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/config.local.git b/config.local.git new file mode 100644 index 0000000..2abb190 --- /dev/null +++ b/config.local.git @@ -0,0 +1,66 @@ +#!/hint/bash + +VCS=git + +PKGREPOS=( + core core-staging core-testing + extra extra-staging extra-testing + multilib multilib-staging multilib-testing + kde-unstable gnome-unstable +) +DEBUGREPOS=( + core-debug core-staging-debug core-testing-debug + extra-debug extra-staging-debug extra-testing-debug + multilib-debug multilib-staging-debug multilib-testing-debug + kde-unstable-debug gnome-unstable-debug +) +TESTING_REPOS=(core-testing extra-testing multilib-testing) +STABLE_REPOS=(core extra) + +ACL_CORE_LIMITED=( + core-staging core-staging-debug + core-testing core-testing-debug +) +ACL_CORE_ALL=( + core core-debug + "${ACL_CORE_LIMITED[@]}" +) +ACL_EXTRA_LIMITED=( + extra-staging extra-staging-debug + extra-testing extra-testing-debug +) +ACL_EXTRA_ALL=( + extra extra-debug + "${ACL_EXTRA_LIMITED[@]}" +) +ACL_DESKTOP=( + kde-unstable kde-unstable-debug + gnome-unstable gnome-unstable-debug +) +ACL_MULTILIB=( + multilib multilib-debug + multilib-staging multilib-staging-debug + multilib-testing multilib-testing-debug +) +ACL=( + [packager]="${ACL_EXTRA_ALL[@]}" + [junior-packager]="${ACL_EXTRA_LIMITED[@]}" + [dev]="${ACL_CORE_ALL[@]} ${ACL_EXTRA_ALL[@]} ${ACL_DESKTOP[@]}" + [junior-dev]="${ACL_CORE_LIMITED[@]} ${ACL_EXTRA_ALL[@]} ${ACL_DESKTOP[@]}" + [multilib]="${ACL_MULTILIB[@]}" +) + +PKGPOOL='pool/packages' +DEBUGPKGPOOL='pool/packages-debug' +SRCPOOL='sources/packages' + +CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" +TMPDIR="/srv/repos/svn-packages/tmp" + +KEYRING="/etc/pacman.d/gnupg" +GIT_PACKAGING_REPOS_URL="https://gitlab.archlinux.org/archlinux/packaging/packages" +GIT_STATE_REPO="/srv/repos/state" +GITUSER="" + +GIT_PACKAGES_CACHE="/srv/repos/pkg-cache" diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 5980703..f9ff780 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -79,7 +79,7 @@ for repo in "${PKGREPOS[@]}"; do # Get the sources from svn mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - export_from_vcs "${pkgbase}" "repos/${repo}-${pkgarch}" "" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" + export_from_vcs "${pkgbase}" "${pkgver}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" if (( $? >= 1 )); then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue diff --git a/db-functions b/db-functions index 35f080c..203891c 100644 --- a/db-functions +++ b/db-functions @@ -42,6 +42,9 @@ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() REPO_MODIFIED=0 +# set static gnupg directory for trusted keyring used by GnuPG, Git etc. +export GNUPGHOME="${KEYRING}" + script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then @@ -265,11 +268,16 @@ getpkgdesc() { # here be dragons is_debug_package() { local pkgfile=${1} - local pkgbase="$(getpkgbase "${pkgfile}")" - local pkgname="$(getpkgname "${pkgfile}")" - local pkgdesc="$(getpkgdesc "${pkgfile}")" + local pkgbase pkgname pkgdesc - [[ ${pkgdesc} == "Detached debugging symbols for "* && ${pkgbase}-debug = ${pkgname} ]] + if ! pkgbase="$(getpkgbase "${pkgfile}")" || \ + ! pkgname="$(getpkgname "${pkgfile}")" || \ + ! pkgdesc="$(getpkgdesc "${pkgfile}")"; then + error "Failed to get PKGINFO metadata from package '%s'" "${pkgfile}" + exit 1 + fi + + [[ ${pkgdesc} == "Detached debugging symbols for "* && ${pkgbase}-debug = "${pkgname}" ]] } check_packager() { @@ -290,6 +298,44 @@ check_builddir() { [[ -n $_builddir && $_builddir = '/build' ]] } +check_author() { + local author + if ! [[ -f "${AUTHORS}" ]]; then + die "No such authors file: '%s'" "${AUTHORS}" + return 1 + fi + if ! author=$(get_author); then + return 1 + fi + return 0 +} + +get_author() { + if ! username=$(/usr/bin/id -un); then + die "Failed to get username from 'id'" + fi + if ! author=$(grep -E " ${username}\$" "${AUTHORS}"); then + die "Missing author information for %s in '%s'" "${username}" "${AUTHORS}" + fi + echo "${author}" +} + +get_author_name() { + local author=$1 + if ! name=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\1/'); then + die "Failed to get name from author line" + fi + echo "${name}" +} + +get_author_email() { + local author=$1 + if ! email=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\2/'); then + die "Failed to get name from author line" + fi + echo "${email}" +} + # Non fatal getpkgfile expanding globs maybe_getpkgfile() { if (( $# != 1 )); then @@ -357,7 +403,7 @@ check_pkgvcs() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 local vcsver vcsnames=() - read -rd'\n' vcsver vcsnames < <(source_pkgbuild "${_pkgbase}" "repos/${repo}-${_pkgarch}"; \ + read -rd'\n' vcsver vcsnames < <(source_pkgbuild "${_pkgbase}" "${_pkgver}"; \ get_full_version; echo "${pkgname[@]}") read -ra vcsnames <<<"${vcsnames}" @@ -382,7 +428,8 @@ check_splitpkgs() { local _pkgbase="$(getpkgbase "${pkgfile}")" local _pkgname="$(getpkgname "${pkgfile}")" local _pkgarch="$(getpkgarch "${pkgfile}")" - local vcsnames=($(source_pkgbuild "${_pkgbase}" "repos/${repo}-${_pkgarch}"; echo "${pkgname[@]}")) + local _pkgver="$(getpkgver "${pkgfile}")" + local vcsnames=($(source_pkgbuild "${_pkgbase}" "${_pkgver}"; echo "${pkgname[@]}")) # not a split package (( ${#vcsnames[@]} > 1 )) || continue @@ -390,6 +437,7 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" + printf '%s\n' "${vcsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/vcs" done popd >/dev/null @@ -484,7 +532,15 @@ check_repo_permission() { [[ -f ${dir}${repo}${FILESEXT} && ! -w ${dir}${repo}${FILESEXT} ]] && return 1 done - return 0 + # Check unix group against ACL + for group in $(/usr/bin/groups); do + [ "${ACL[${group}]+exists}" ] || continue + read -r -a acl_repos <<< "${ACL[${group}]}" + in_array "${repo}" "${acl_repos[@]}" || continue + return 0 + done + + return 1 } set_repo_permission() { @@ -546,4 +602,28 @@ check_reproducible() { done } +# TODO: Needs to verify the keyid is in the keyring +check_signed_tag(){ + local pkgbase="$(getpkgbase ${1})" + local pkgver="$(gittag_from_pkgver "$(getpkgver "${1}")")" + if ! git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" verify-tag "${pkgver}" >/dev/null 2>&1; then + return 1 + fi +} + +# Verified the package .BUIDINFO PKGBUID sha256sum against the checked out version +check_pkgbuild_checksum(){ + local pkgbase="$(getpkgbase ${1})" + local pkgver="$(getpkgver ${1})" + local pkgfile_checksum="$(_grep_buildinfo "${1}" "pkgbuild_sha256sum")" + + local gittag="$(gittag_from_pkgver "$(getpkgver "${1}")")" + + local sum="$(sha256sum <(git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" show "${gittag}":PKGBUILD))" + if [[ "$pkgfile_checksum" != "${sum%% *}" ]]; then + return 1 + fi +} + + . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-functions-git b/db-functions-git new file mode 100644 index 0000000..cbfa9fe --- /dev/null +++ b/db-functions-git @@ -0,0 +1,188 @@ +#!/hint/bash + +if [[ -n ${GITUSER} ]]; then + setfacl -m u:"${GITUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${GITUSER}":rwx "${WORKDIR}" +fi + +# unset any behavior influencing environment variables +# https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables +while read -r var; do + unset "${var}"; +done < <(env | grep -E '^GIT' | awk -F= '{print $1}') +unset PREFIX +unset EMAIL + + +# Converts from the PKGBUILD tag to the git repository tag +# Input 1:1.0~0-1 +# Output 1-1.0.0-1 +gittag_from_pkgver() { + local pkgver="$1" + local gittag + gittag="${pkgver/:/-}" + gittag="${gittag//\~/.}" + printf "%s" "${gittag}" +} + +arch_git() { + if [[ -z ${GITUSER} ]]; then + /usr/bin/git "${@}" + else + sudo -u "${GITUSER}" -- /usr/bin/git "${@}" + fi +} + +# 1. replace single '+' between word boundaries with '-' +# 2. replace any other '+' with literal 'plus' +# 3. replace any special chars other than '_', '-' and '.' with '-' +gitlab_project_name_to_path() { + local name=$1 + printf "%s" "${name}" \ + | sed -E 's/([a-zA-Z0-9]+)\+([a-zA-Z]+)/\1-\2/g' \ + | sed -E 's/\+/plus/g' \ + | sed -E 's/[^a-zA-Z0-9_\-\.]/-/g' +} + +# Fetch the package sources into a global cache +fetch_pkgbuild() { + local pkgbase="${1}" + local project_path src target + project_path=$(gitlab_project_name_to_path "${pkgbase}") + src="${GIT_PACKAGING_REPOS_URL}/${project_path}.git" + target="${GIT_PACKAGES_CACHE}/${pkgbase}" + if [[ ! -d $target ]]; then + if ! arch_git -c core.sharedRepository=group clone --origin origin --bare --mirror "${src}" "${target}"; then + return 1 + fi + return 0 + fi + if ! arch_git -C "${target}" fetch --prune --prune-tags; then + return 1 + fi + return 0 +} + + +# Source the PKGBUILD from the package's git/svn/whatever repo. +source_pkgbuild() { + local pkgbase="$1" + local tag=$(gittag_from_pkgver "${2}") + + . <(arch_git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" show "${tag}":PKGBUILD 2>/dev/null || echo false) +} + +# Export PKGBUILD resource following the same rules as source_pkgbuild() +export_from_vcs() { + local pkgbase="$1" + local tag=$(gittag_from_pkgver "${2}") + local dest="$3" + + if [[ ! -e ${dest} ]]; then + mkdir -p "${dest}" + arch_git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" + fi +} + +# Which repo is this package in? +find_repo_for_package() { + local pkgbase=${1} + local pkgarch=${2} + local candidates=("${@:3}") + + local repos=($(arch_git -C "${GIT_STATE_REPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ + grep -xFf <(printf "%s\n" "${candidates[@]/%/-${pkgarch}}" "${candidates[@]/%/-any}"))) + + if (( ${#repos[@]} > 1 )); then + die "%s is present in multiple repos (%s)" "${pkgbase}" "${repos[*]}" + fi + (( ${#repos[@]} == 1 )) || return $? + + printf '%s\n' "${repos[@]%/}" +} + +# Get the current pkgver from a given repo +pkgver_from_state_repo() { + local pkgbase=${1} + local repo=${2} + property_from_state_repo "${pkgbase}" "${repo}" 2 +} + +# Get the current git tag name from a given repo +gittag_from_state_repo() { + local pkgbase=${1} + local repo=${2} + property_from_state_repo "${pkgbase}" "${repo}" 3 +} + +# Get the given property position from a given repo +property_from_state_repo() { + local pkgbase=${1} + local repo=${2} + local prop=${3} + awk '{print $'"${prop}"'}' "${GIT_STATE_REPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 +} + +# Commit changes staged by (successive?) vcs_(re)?move_package runs. +vcs_commit() { + local commit_message=$1 + local username author name email + + if ! username=$(/usr/bin/id -un); then + die "Failed to get username from 'id'" + fi + if ! author=$(grep -E " ${username}\$" "${AUTHORS}"); then + die "Missing author information for %s in '%s'" "${username}" "${AUTHORS}" + fi + if ! name=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\1/'); then + die "Failed to extract name from author line" + fi + if ! email=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\2/'); then + die "Failed to extract email from author line" + fi + + arch_git -c user.name="${name}" -c user.email="${email}" -C "${GIT_STATE_REPO}" commit -m "${commit_message}" +} + + +vcs_update_package() { + local pkgbase="$1" + local pkgver="$2" + local dest="$3" + local gittag + gittag="$(gittag_from_pkgver "${pkgver}")" + + mkdir -p "${GIT_STATE_REPO}/${dest}" + printf '%s %s %s %s\n' \ + "${pkgbase}" \ + "${pkgver}" \ + "${gittag}" \ + "$(git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" rev-parse "${gittag}")" \ + > "${GIT_STATE_REPO}/${dest}/${pkgbase}" + + arch_git -C "${GIT_STATE_REPO}" add "${GIT_STATE_REPO}/${dest}/${pkgbase}" + vcs_commit "update ${pkgbase} to ${pkgver} in ${dest}" +} + +# Write to the VCS in order to track a package moving between different pacman +# repositories. +vcs_move_package() { + local pkgbase=${1} + local vcsrepo_from=${2} + local vcsrepo_to=${3} + + mkdir -p "${GIT_STATE_REPO}/${vcsrepo_to}" + arch_git -C "${GIT_STATE_REPO}" mv --force "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" + vcs_commit "move ${pkgbase} from ${vcsrepo_from} to ${vcsrepo_to}" +} + +# Write to the VCS in order to track a package being deleted from a pacman +# repository. +vcs_remove_package() { + local pkgbase=${1} + local vcsrepo=${2} + + arch_git -C "${GIT_STATE_REPO}" rm "${vcsrepo}/${pkgbase}" + vcs_commit "remove ${pkgbase} from ${vcsrepo}" +} diff --git a/db-move b/db-move index 76e4f1e..9695b8c 100755 --- a/db-move +++ b/db-move @@ -31,6 +31,10 @@ if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; t die "You don't have permission to move packages from %s to %s" "$repo_from" "$repo_to" fi +if ! check_author; then + die "You don't have a matching author mapping" +fi + # TODO: this might lock too much (architectures) for pkgarch in "${ARCHES[@]}"; do repo_lock "${repo_to}" "${pkgarch}" || exit 1 @@ -42,11 +46,15 @@ for pkgbase in "${args[@]:2}"; do found=false for pkgarch in "${ARCHES[@]}"; do if vcsrepo_from=$(find_repo_for_package "${pkgbase}" "${pkgarch}" "${repo_from}"); then - #FIXME: abort if PKGBUILD not there - read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "repos/${vcsrepo_from}"; \ - get_full_version; echo "${pkgname[@]}") + if ! _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo_from}"); then + die "%s has no pkgver entry in %s" "${pkgbase}" "${vcsrepo_from}" + fi + if ! fetch_pkgbuild "${pkgbase}"; then + die "Couldn't find package %s in git!" "${pkgbase}" + fi + read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "${_pkgver}"; \ + get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" - if (( ${#pkgnames[@]} < 1 )); then die "Could not read pkgname" fi @@ -71,6 +79,7 @@ done msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" +declare -A pkgbase_arches=() for arch in "${ARCHES[@]}"; do declare -a add_pkgs_$arch declare -a add_debug_pkgs_$arch @@ -89,12 +98,15 @@ for pkgbase in "${args[@]:2}"; do tarches=("${pkgarch}") fi msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "repos/${repo_from}-${pkgarch}"; \ + if ! _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo_from}"); then + die "%s has no entry in %s" "${pkgbase}" "${vcsrepo_from}" + fi + read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "${_pkgver}"; \ get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" - vcs_move_package "${pkgbase}" "${repo_from}-${pkgarch}" "${repo_to}-${pkgarch}" tag_list+=", $pkgarch" + pkgbase_arches[$pkgbase]+="$pkgarch " for tarch in "${tarches[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" @@ -128,7 +140,6 @@ for pkgbase in "${args[@]:2}"; do fi done tag_list="${tag_list#, }" - vcs_commit "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done for tarch in "${ARCHES[@]}"; do @@ -146,6 +157,19 @@ for tarch in "${ARCHES[@]}"; do fi done +# Only modify repository if everything else has been done +# We really don't want it to represent a broken state +if ((REPO_MODIFIED)); then + for pkgbase in "${args[@]:2}"; do + read -r -a tarches <<< "${pkgbase_arches[${pkgbase}]}" + for tarch in "${tarches[@]}"; do + if ! vcs_move_package "${pkgbase}" "${repo_from}-${tarch}" "${repo_to}-${tarch}"; then + die "Couldn't move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" + fi + done + done +fi + for pkgarch in "${ARCHES[@]}"; do repo_unlock "${repo_from}" "${pkgarch}" repo_unlock "${repo_to}" "${pkgarch}" diff --git a/db-remove b/db-remove index b855e5b..007efba 100755 --- a/db-remove +++ b/db-remove @@ -17,6 +17,9 @@ vcsrepo="$repo-$arch" if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" fi +if ! check_author; then + die "You don't have a matching author mapping" +fi if [[ $arch = any ]]; then tarches=("${ARCHES[@]}") @@ -28,18 +31,20 @@ for tarch in "${tarches[@]}"; do repo_lock "$repo" "$tarch" || exit 1 done +remove_pkgbases=() remove_pkgs=() remove_debug_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - if remove_pkgs+=($(source_pkgbuild "${pkgbase}" "repos/${vcsrepo}" && echo "${pkgname[@]}")); then - vcs_remove_package "${pkgbase}" "${vcsrepo}" - vcs_commit "${0##*/}: $pkgbase removed by $(id -un)" + if fetch_pkgbuild "${pkgbase}" && \ + _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo}") && \ + remove_pkgs+=($(source_pkgbuild "${pkgbase}" "${_pkgver}" && printf "%s\n" "${pkgname[@]}")); then + remove_pkgbases+=("${pkgbase}") else - warning "%s not found in %s" "$pkgbase" "$vcsrepo" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" + warning "pkgbase %s not found in %s" "$pkgbase" "$vcsrepo" + warning "Removing only pkgname %s from the repo" "$pkgbase" + warning "If it was a split package you have to pass its pkgbase to remove it completely!" remove_pkgs+=("$pkgbase") fi if is_globfile "${FTP_BASE}/${repo}-debug/os/${tarch}/${pkgbase}-debug"*; then @@ -55,5 +60,15 @@ for tarch in "${tarches[@]}"; do if (( ${#remove_debug_pkgs[@]} >= 1 )); then arch_repo_modify remove "${repo}-debug" "${tarch}" "${remove_debug_pkgs[@]}" fi +done + +if ((REPO_MODIFIED)); then + for pkgbase in "${remove_pkgbases[@]}"; do + vcs_remove_package "${pkgbase}" "${vcsrepo}" + done +fi + +# Remove all the locks we created +for tarch in "${tarches[@]}"; do repo_unlock "$repo" "$tarch" done diff --git a/db-update b/db-update index 4156824..b9b0537 100755 --- a/db-update +++ b/db-update @@ -29,21 +29,42 @@ for repo in "${repos[@]}"; do done done + +for repo in "${repos[@]}"; do + if pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})); then + for pkg in "${pkgs[@]}"; do + if ! fetch_pkgbuild "$(getpkgbase "${pkg}")"; then + die "Couldn't find package %s in git!" "${pkg}" + fi + done + fi +done + + +if ! check_author; then + die "You don't have a matching author mapping" +fi + # check if packages are valid for repo in "${repos[@]}"; do if ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in %s" "$repo" fi - pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) - if (( $? == 0 )); then + if pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})); then for pkg in "${pkgs[@]}"; do if [[ -h ${pkg} ]]; then die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi + if ! check_signed_tag "${pkg}"; then + die "Package %s does not have a signed tag matching the version" "$repo/${pkg##*/}" + fi + if ! check_pkgbuild_checksum "${pkg}"; then + die "Package %s was not built with the checked in PKGBUILD" "$repo/${pkg##*/}" + fi if ! check_pkgfile "${pkg}"; then die "Package %s is not consistent with its meta data" "$repo/${pkg##*/}" fi - if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then + if ! pacman-key --verify "${pkg}.sig" "${pkg}" >/dev/null 2>&1; then die "Package %s does not have a valid signature" "$repo/${pkg##*/}" fi if ! check_pkgvcs "${pkg}" "${repo}"; then @@ -64,14 +85,16 @@ for repo in "${repos[@]}"; do if ! check_builddir "${pkg}"; then die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi - if ! check_reproducible "${pkg}"; then - error "Package %s is not reproducible." "${pkg}" - die "Ensure that all dependencies are available in the repositories or are added in the same db-update." - fi + #if ! check_reproducible "${pkg}"; then + # error "Package %s is not reproducible." "${pkg}" + # die "Ensure that all dependencies are available in the repositories or are added in the same db-update." + #fi done if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" fi + # TODO: + # Add history repo information check as detailed in proposal else die "Could not read %s" "$STAGING" fi @@ -79,6 +102,7 @@ done for repo in "${repos[@]}"; do msg "Updating [%s]..." "$repo" + declare -A add_vcspkgs any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() @@ -86,8 +110,19 @@ for repo in "${repos[@]}"; do arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" + if [[ ! "${add_vcspkgs["${pkgfile}::pkgbase"]+exists}" ]]; then + if ! pkgbase="$(getpkgbase "${pkg}")" || \ + ! pkgver="$(getpkgver "${pkg}")"; then + die "Failed to get metadata from '%s'" "${pkg}" + fi + add_vcspkgs["${pkgfile}::pkgbase"]="${pkgbase}" + add_vcspkgs["${pkgfile}::pkgver"]="${pkgver}" + + is_debug_package "${pkg}" + add_vcspkgs["${pkgfile}::debug"]=$(( $? == 0 )) + fi - if is_debug_package "${pkg}"; then + if (( ${add_vcspkgs["${pkgfile}::debug"]} )); then debug_pkgs+=("${pkgfile}") currentpool=${PKGPOOL}-debug currentrepo=${repo}-debug @@ -102,12 +137,13 @@ for repo in "${repos[@]}"; do if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${currentpool}" fi + mkdir -p "$FTP_BASE/${currentrepo}/os/${pkgarch}" ln -sf "../../../${currentpool}/${pkgfile}" "$FTP_BASE/${currentrepo}/os/${pkgarch}" # also move signatures if [[ -f ${pkg}.sig ]]; then mv "${pkg}.sig" "$FTP_BASE/${currentpool}" fi - if [[ ${PKGPOOL} = ${currentpool} ]]; then + if [[ ${PKGPOOL} = "${currentpool}" ]]; then # do not archive debug info, this is not of historic interest "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" fi @@ -121,6 +157,16 @@ for repo in "${repos[@]}"; do if (( ${#debug_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}-debug" "${pkgarch}" "${debug_pkgs[@]}" fi + if ((REPO_MODIFIED)); then + for pkg in "${add_pkgs[@]}"; do + if [[ $pkg == *-any${PKGEXTS} ]]; then + pkgarch=any + fi + pkgbase="${add_vcspkgs[$pkg::pkgbase]}" + pkgver="${add_vcspkgs[$pkg::pkgver]}" + vcs_update_package "${pkgbase}" "${pkgver}" "${repo}-${pkgarch}" + done + fi done done diff --git a/test/Dockerfile b/test/Dockerfile index 7353eb8..8bdca9e 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,8 +1,20 @@ FROM docker.io/archlinux/archlinux -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils -RUN pacman-key --init -RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -RUN useradd -N -g users -G wheel -d /build -m tester +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils git && \ + pacman-key --init && \ + echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel && \ + useradd -N -g users -G wheel -d /build -m tester && \ + ln -sf /dbscripts/db-archive /usr/local/bin/ && \ + ln -sf /dbscripts/db-move /usr/local/bin/ && \ + ln -sf /dbscripts/db-remove /usr/local/bin/ && \ + ln -sf /dbscripts/db-repo-add /usr/local/bin/ && \ + ln -sf /dbscripts/db-repo-remove /usr/local/bin/ && \ + ln -sf /dbscripts/db-update /usr/local/bin/ && \ + ln -sf /dbscripts/testing2x /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/devlist-mailer /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/ftpdir-cleanup /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/integrity-check /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/sourceballs /usr/local/bin/ + USER tester RUN echo -e "\ Key-Type: RSA\n\ @@ -13,7 +25,7 @@ Name-Email: tester@localhost\n\ Expire-Date: 0\n\ %no-protection\n\ %commit\n"\ -| gpg --quiet --batch --no-tty --no-permission-warning --gen-key -RUN gpg --export | sudo pacman-key -a - -RUN sudo pacman-key --lsign-key tester@localhost +| gpg --quiet --batch --no-tty --no-permission-warning --gen-key && \ + gpg --export | sudo pacman-key -a - && \ + sudo pacman-key --lsign-key tester@localhost ENV PACKAGER="Bob Tester " diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 3dd0556..e214198 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -193,3 +193,70 @@ load ../lib/common checkPackage extra pkg-split-debuginfo 1-1 checkPackage extra-debug pkg-split-debuginfo 1-1 } + +@test "move package with insufficient target repo permissions fails" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + run db-move testing noperm pkg-simple-a pkg-simple-b + [ "$status" -ne 0 ] + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage noperm ${pkgbase} + checkPackage testing ${pkgbase} 1-1 + done +} + +@test "move package with insufficient source repo permissions fails" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage noperm ${pkgbase} + done + + enablePermission noperm + db-update + disablePermissionOverride + + run db-move noperm testing pkg-simple-a pkg-simple-b + [ "$status" -ne 0 ] + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage testing ${pkgbase} + checkPackage noperm ${pkgbase} 1-1 + done +} + +@test "move package with author mapping" { + releasePackage testing pkg-any-a + db-update + + db-move testing extra pkg-any-a + + checkPackage extra pkg-any-a 1-1 + checkRemovedPackage testing pkg-any-a + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "move package with missing author mapping fails" { + releasePackage testing pkg-any-a + db-update + + emptyAuthorsFile + run db-move testing extra pkg-any-a + [ "$status" -ne 0 ] + + checkPackage testing pkg-any-a 1-1 + checkRemovedPackage extra pkg-any-a +} diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 484789a..18e0e26 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -132,3 +132,39 @@ load ../lib/common checkRemovedPackage extra ${pkgbase} done } + +@test "remove package with insufficient repo permissions fails" { + local pkgbase='pkg-any-a' + + releasePackage noperm ${pkgbase} + + enablePermission noperm + db-update + disablePermissionOverride + + run db-remove noperm any ${pkgbase} + [ "$status" -ne 0 ] + + checkPackage noperm ${pkgbase} 1-1 +} + +@test "remove package with author mapping" { + releasePackage testing pkg-any-a + db-update + + db-remove testing any pkg-any-a + + checkRemovedPackage testing pkg-any-a + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "remove package with missing author mapping fails" { + releasePackage testing pkg-any-a + db-update + + emptyAuthorsFile + run db-remove testing any pkg-any-a + [ "$status" -ne 0 ] + + checkPackage testing pkg-any-a 1-1 +} diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 8d0851e..4db8962 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -245,17 +245,26 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-a' } -@test "add package with inconsistent pkgbuild fails" { +@test "add package with inconsistent pkgbuild in branch succeeds" { releasePackage extra 'pkg-any-a' updateRepoPKGBUILD 'pkg-any-a' extra any + db-update + checkPackage extra 'pkg-any-a' 1-1 +} + +@test "add package with inconsistent pkgbuild in tag fails" { + releasePackage extra 'pkg-any-a' + + retagModifiedPKGBUILD 'pkg-any-a' + run db-update [ "$status" -ne 0 ] checkRemovedPackageDB extra 'pkg-any-a' } -@test "add package with insufficient permissions fails" { +@test "add package with insufficient directory permissions fails" { releasePackage core 'pkg-any-a' releasePackage extra 'pkg-any-b' @@ -268,6 +277,17 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-b' } +@test "add package with insufficient repo permissions fails" { + releasePackage noperm 'pkg-any-a' + releasePackage extra 'pkg-any-b' + + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB noperm 'pkg-any-a' + checkRemovedPackageDB extra 'pkg-any-b' +} + @test "package has to be aregular file" { local p local target=$(mktemp -d) @@ -310,3 +330,22 @@ load ../lib/common checkPackage extra-debug ${pkgbase} 1-1 done } + +@test "add package with author mapping" { + releasePackage extra pkg-any-a + + db-update + + checkPackage extra pkg-any-a 1-1 + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "add package with missing author mapping fails" { + releasePackage extra pkg-any-a + + emptyAuthorsFile + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackage extra pkg-any-a +} diff --git a/test/lib/common.bash b/test/lib/common.bash index b66ec95..953b2e1 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,12 +1,44 @@ . /usr/share/makepkg/util.sh shopt -s extglob + +# Copy of db-functions-git +arch_git() { + if [[ -z ${GITUSER} ]]; then + /usr/bin/git "${@}" + else + sudo -u "${GITUSER}" -- /usr/bin/git "${@}" + fi +} + __updatePKGBUILD() { local pkgrel pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=${pkgrel}" + git add . + git commit -m "update pkg to pkgrel=${pkgrel}" + git push +} + +__retagModifiedPKGBUILD() { + local pkgver + local pkgrel + local gittag + + pkgver=$(. PKGBUILD; echo "${pkgrel}") + pkgrel=$(. PKGBUILD; echo "${pkgrel}") + gittag="${pkgver}-${pkgrel}" + + echo >> PKGBUILD + git add PKGBUILD + git commit -m "modified PKGBUILD" + + # re-tag + git push origin :"${gittag}" + git tag -d "${gittag}" + git tag -s -m "released ${gittag}" "${gittag}" + git push --tags origin main } __getCheckSum() { @@ -15,6 +47,18 @@ __getCheckSum() { echo "${result%% *}" } +# Converts from the git repository tag to the PKGBUILD tag +# Input 1-1.0.0-1 +# Output 1:1.0.0-1 +__parseGitTag(){ + tag="${1}" + while IFS=- read -r pkgrel pkgver epoch; do + test -n "${epoch}" && printf "%s:" "$epoch" + printf "%s" "$(echo "$pkgver" | rev)" + printf "%s" "$(echo "$pkgrel-" | rev)" + done < <(echo "${tag}" | rev) +} + # Proxy function to check if a file exists. Using [[ -f ... ]] directly is not # always wanted because we might want to expand bash globs first. This way we # can pass unquoted globs to __isGlobfile() and have them expanded as function @@ -68,23 +112,24 @@ __archrelease() { local pkgarches local tarch local tag - - pkgarches=($(. PKGBUILD; echo ${arch[@]})) - pushd .. - for tarch in ${pkgarches[@]}; do - tag=${repo}-${tarch} - - if [[ -d repos/$tag ]]; then - svn rm repos/$tag/PKGBUILD - else - mkdir -p repos/$tag - svn add repos/$tag + local rev + local head + + pkgver=$(. PKGBUILD; get_full_version) + gittag=${pkgver/:/-} + + # avoid trying to tag the same commit twice + if rev=$(git rev-list -n1 "$gittag" 2>/dev/null); then + head=$(git rev-parse HEAD) + if [[ "$rev" != "$head" ]]; then + error "failed to tag revision %s" "${head}" + error "tag '%s' already exists for revision %s" "${gittag}" "${rev}" + exit 1 fi - - svn copy -r HEAD trunk/PKGBUILD repos/$tag/ - done - svn commit -m "__archrelease" - popd + return 0 + fi + git tag -s -m "released $pkgbase-$pkgver" "$gittag" + git push --tags origin main } setup() { @@ -92,18 +137,21 @@ setup() { local pkg local r local a + local username PKGEXT=".pkg.tar.xz" TMP="$(mktemp -d)" + chmod 770 "$TMP" export DBSCRIPTS_CONFIG=${TMP}/config.local cat < "${DBSCRIPTS_CONFIG}" FTP_BASE="${TMP}/ftp" ARCHIVE_BASE="${TMP}/archive" ARCHIVEUSER="" - SVNREPO="file://${TMP}/svn-packages-repo" - PKGREPOS=('core' 'extra' 'testing' 'staging') + PKGREPOS=('core' 'extra' 'testing' 'staging' 'noperm') DEBUGREPOS=('core-debug' 'extra-debug' 'testing-debug' 'staging-debug') + ACL=([users]="extra core staging testing core-debug extra-debug testing-debug staging-debug") + PKGPOOL='pool/packages' DEBUGPKGPOOL='pool/packages-debug' SRCPOOL='sources/packages' @@ -117,10 +165,41 @@ setup() { ARCHES=(x86_64 i686) CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false + VCS=git + KEYRING="/etc/pacman.d/gnupg" + GIT_PACKAGING_REPOS_URL="${TMP}/git-packages" + GIT_STATE_REPO="${TMP}/repository" + GIT_PACKAGES_CACHE="${TMP}/git-pkg-repos" + GITUSER="" + AUTHORS="${TMP}/authors.conf" + + if [[ -f "${TMP}/config.override" ]]; then + . "${TMP}/config.override" + fi eot + + username=$(/usr/bin/id -un) + cat < "${TMP}/authors.conf" + qux dux + Cake Foobar ${username} + muh cow + ${username} <${username}@yay> doo +eot + . config - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + git config --global user.email "tester@localhost" + git config --global user.name "Bob Tester" + git config --global init.defaultBranch main + git config --global advice.detachedHead false + + + # This is for our git clones when initializing bare repos + TMP_WORKDIR_GIT=${TMP}/git-clones + + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup} + mkdir -p "${GIT_PACKAGING_REPOS_URL}" + mkdir -p "${TMP_WORKDIR_GIT}" for r in ${PKGREPOS[@]}; do mkdir -p "${TMP}"/staging/${r}{,-debug} @@ -144,35 +223,60 @@ eot touch "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}/${line}"{,.sig} done - svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + git init --bare --shared=group "${TMPDIR}/git-packages-bare.git" + mkdir "${GIT_STATE_REPO}" + chmod 777 "${GIT_STATE_REPO}" + arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GIT_STATE_REPO}" 2>/dev/null } teardown() { rm -rf "${TMP}" } +enablePermission() { + local repo=$1 + grep ACL "${TMP}/config.local" | sed 's/")/ '"$repo"'")/' > "${TMP}/config.override" +} + +disablePermissionOverride() { + rm -f "${TMP}/config.override" +} + releasePackage() { local repo=$1 local pkgbase=$2 - if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then - mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} - cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkgbase} - svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" + if [ ! -d "${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" ]; then + git init --bare --shared=all "${GIT_PACKAGING_REPOS_URL}/${pkgbase}".git + git -c "core.sharedRepository=group" clone "${GIT_PACKAGING_REPOS_URL}/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" + cp -r "fixtures/${pkgbase}"/* "${TMP_WORKDIR_GIT}/${pkgbase}" + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" add "${TMP_WORKDIR_GIT}/${pkgbase}"/* + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" commit -m "initial commit of ${pkgbase}" + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" push + + fi + + if [ ! -d "${TMP_WORKDIR_GIT}/${pkgbase}" ]; then + git clone --origin origin "${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" fi - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + git pull origin main __buildPackage "${STAGING}"/${repo} __archrelease ${repo} + chmod -R 777 "${GIT_PACKAGING_REPOS_URL}/" popd } +emptyAuthorsFile() { + echo > "${TMP}/authors.conf" +} + updatePackage() { local pkgbase=$1 - pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + git pull origin main __updatePKGBUILD __buildPackage popd @@ -183,11 +287,19 @@ updateRepoPKGBUILD() { local repo=$2 local arch=$3 - pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" __updatePKGBUILD popd } +retagModifiedPKGBUILD() { + local pkgbase=$1 + + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + __retagModifiedPKGBUILD + popd +} + checkPackageDB() { local repo=$1 local pkgbase=$2 @@ -252,19 +364,20 @@ checkPackage() { local pkgbase=$2 local pkgver=$3 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - local dirarches=() pkgbuildarches=() local pkgbuild dirarch pkgbuildver - for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; do + for pkgbuild in "${GIT_STATE_REPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; do [[ -e $pkgbuild ]] || continue - dirarch=${pkgbuild%/PKGBUILD} + dirarch=${pkgbuild%/${pkgbase}} dirarch=${dirarch##*-} dirarches+=("$dirarch") - pkgbuildarches+=($(. "$pkgbuild"; echo ${arch[@]})) - pkgbuildver=$(. "$pkgbuild"; get_full_version) - [[ $pkgver = "$pkgbuildver" ]] + pkgbuildarches+=($(. "${TMP_WORKDIR_GIT}/${pkgbase}/PKGBUILD"; echo ${arch[@]})) + + while read -r _ tag _; do + pkgbuildver=$(__parseGitTag "$tag") + [[ $pkgver = "$pkgbuildver" ]] + done < "$pkgbuild" done # Verify that the arches-from-dirnames and # arches-from-PKGBUILDs agree (that a PKGBUILD existed for @@ -282,8 +395,7 @@ checkRemovedPackage() { local repo=$1 local pkgbase=$2 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; then + if __isGlobfile "${GIT_STATE_REPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; then return 1 fi @@ -329,3 +441,17 @@ checkRemovedPackageDB() { done done } + +checkStateRepoAutoredBy() { + local expected=$1 + local author + + if ! author=$(git -C "${GIT_STATE_REPO}" show -s --format='%an <%ae>' HEAD); then + die 'Failed to query author of state repository' + fi + if [[ "${expected}" != "${author}" ]]; then + error "Author doesn't match, expected: '%s', actual: '%s'" "${expected}" "${author}" + return 1 + fi + return 0 +}