From 9894b40f093ef1d18c9692f39dfee897b9c7a09a Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:37:28 -0400 Subject: [PATCH 01/20] wip: debug: lock/unlock both repos at once --- db-functions | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index d37b680..fdf535a 100644 --- a/db-functions +++ b/db-functions @@ -109,7 +109,14 @@ trap cleanup EXIT #repo_lock [timeout] -repo_lock () { +repo_lock() { + local repo base=${1}; shift + for repo in ${base} ${base}-debug; do + _repo_lock ${repo} "${@}" + done +} + +_repo_lock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" local DBLOCKFILE="${FTP_BASE}/${1}/os/${2}/${1}${DBEXT}.lck" local _count @@ -152,7 +159,15 @@ repo_lock () { return 1 } -repo_unlock () { #repo_unlock +#repo_unlock +repo_unlock () { + local repo base=${1}; shift + for repo in ${base} ${base}-debug; do + _repo_unlock ${repo} "${@}" + done +} + +_repo_unlock () { local LOCKDIR="$TMPDIR/.repolock.$1.$2" if [[ ! -d $LOCKDIR ]]; then warning "Repo lock [%s] (%s) was not locked!" "$1" "$2" From cb20b63a0cad9d513a8874cdb5979622f6349b2a Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 14 Jun 2020 15:59:52 -0400 Subject: [PATCH 02/20] wip: add support for debug packages --- db-functions | 11 ++++++++++- db-update | 31 ++++++++++++++++++++++++------- test/lib/common.bash | 4 ++-- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index fdf535a..6902913 100644 --- a/db-functions +++ b/db-functions @@ -249,6 +249,14 @@ getpkgarch() { echo "$_ver" } +is_debug_package() { + local pkgfile=${1} + local pkgbase="$(getpkgbase "${pkgfile}")" + local pkgname="$(getpkgname "${pkgfile}")" + + [[ ${pkgbase}-debug = ${pkgname} ]] +} + check_packager() { local _packager @@ -325,7 +333,7 @@ check_pkgvcs() { read -ra vcsnames <<<"${vcsnames}" [[ "${vcsver}" = "${_pkgver}" ]] || return 1 - in_array "${_pkgname}" "${vcsnames[@]}" || return 1 + in_array "${_pkgname}" "${vcsnames[@]}" "${_pkgbase}-debug" || return 1 return 0 } @@ -349,6 +357,7 @@ check_splitpkgs() { # not a split package (( ${#vcsnames[@]} > 1 )) || continue + [[ ${_pkgbase}-debug = ${_pkgname} ]] && continue mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" diff --git a/db-update b/db-update index b85295f..586c17a 100755 --- a/db-update +++ b/db-update @@ -82,28 +82,45 @@ for repo in "${repos[@]}"; do any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() + debug_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgfile="${pkg##*/}" + + if is_debug_package "${pkg}"; then + debug_pkgs+=("${pkgfile}") + currentpool=${PKGPOOL}-debug + currentrepo=${repo}-debug + else + add_pkgs+=("${pkgfile}") + currentpool=${PKGPOOL} + currentrepo=${repo} + fi + msg2 '%s (%s)' "$pkgfile" "$pkgarch" # any packages might have been moved by the previous run if [[ -f ${pkg} ]]; then - mv "${pkg}" "$FTP_BASE/${PKGPOOL}" + mv "${pkg}" "$FTP_BASE/${currentpool}" fi - ln -s "../../../${PKGPOOL}/${pkgfile}" "$FTP_BASE/$repo/os/${pkgarch}" + ln -s "../../../${currentpool}/${pkgfile}" "$FTP_BASE/${currentrepo}/os/${pkgarch}" # also move signatures if [[ -f ${pkg}.sig ]]; then - mv "${pkg}.sig" "$FTP_BASE/${PKGPOOL}" + mv "${pkg}.sig" "$FTP_BASE/${currentpool}" fi - "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" - if [[ -f $FTP_BASE/${PKGPOOL}/${pkgfile}.sig ]]; then - ln -s "../../../${PKGPOOL}/${pkgfile}.sig" "$FTP_BASE/$repo/os/${pkgarch}" + if [[ ${PKGPOOL} = ${currentpool} ]]; then + # do not archive debug info, this is not of historic interest + "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" + fi + if [[ -f $FTP_BASE/${currentpool}/${pkgfile}.sig ]]; then + ln -s "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi - add_pkgs+=("${pkgfile}") done if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi + if (( ${#debug_pkgs[@]} >= 1 )); then + arch_repo_modify add "${repo}-debug" "${pkgarch}" ${debug_pkgs[@]} + fi done done diff --git a/test/lib/common.bash b/test/lib/common.bash index 0bff8e5..44feb4b 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -121,9 +121,9 @@ eot mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} for r in ${PKGREPOS[@]}; do - mkdir -p "${TMP}/staging/${r}" + mkdir -p "${TMP}"/staging/${r}{,-debug} for a in ${ARCHES[@]}; do - mkdir -p "${TMP}/ftp/${r}/os/${a}" + mkdir -p "${TMP}"/ftp/${r}{,-debug}/os/${a} done done mkdir -p "${TMP}/ftp/${PKGPOOL}" From 2a5f244be44d8786d1e2f2f6bc7f1b825c99436a Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Tue, 1 Sep 2020 15:43:39 -0400 Subject: [PATCH 03/20] debug prints --- db-functions | 4 ++++ db-update | 2 ++ test/Dockerfile | 2 +- test/cases/db-move.bats | 6 ++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/db-functions b/db-functions index 6902913..2751543 100644 --- a/db-functions +++ b/db-functions @@ -459,6 +459,7 @@ check_repo_permission() { } set_repo_permission() { + local -; set -x local repo=$1 local arch=$2 local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" @@ -476,11 +477,13 @@ set_repo_permission() { } arch_repo_modify() { + echo "call: arch_repo_modify ${@@Q}" local action=$1 local repo=$2 local arch=$3 local pkgs=("${@:4}") local dbfile="${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT}" + tree "${FTP_BASE}" if [[ ${action} = remove && ! -f ${dbfile} ]]; then error "No database found at '%s'" "$dbfile" @@ -489,6 +492,7 @@ arch_repo_modify() { # package files for repo-add might be relative to repo dir pushd "${dbfile%/*}" >/dev/null + ls /usr/bin/"repo-${action}" -q "${dbfile}" "${pkgs[@]}" \ || error '%s' "repo-${action} ${dbfile@Q} ${pkgs[*]@Q}" set_repo_permission "${repo}" "${arch}" diff --git a/db-update b/db-update index 586c17a..069c564 100755 --- a/db-update +++ b/db-update @@ -79,6 +79,7 @@ done for repo in "${repos[@]}"; do msg "Updating [%s]..." "$repo" + tree "${STAGING}/${repo}" any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() @@ -115,6 +116,7 @@ for repo in "${repos[@]}"; do ln -s "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi done + declare -p add_pkgs debug_pkgs if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" ${add_pkgs[@]} fi diff --git a/test/Dockerfile b/test/Dockerfile index 5dc8199..190f265 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM docker.io/archlinux/archlinux -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index a3b0bb4..56a4331 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -7,15 +7,21 @@ load ../lib/common local arch for pkgbase in ${pkgs[@]}; do + echo "releasing to testing: $pkgbase" releasePackage testing ${pkgbase} done + echo "db-updating..." db-update + echo "db-move testing -> extra pkg-simple-a" db-move testing extra pkg-simple-a + echo checkRemovedPackage testing pkg-simple-a checkRemovedPackage testing pkg-simple-a + echo checkPackage extra pkg-simple-a 1-1 checkPackage extra pkg-simple-a 1-1 + echo checkPackage testing pkg-simple-b 1-1 checkPackage testing pkg-simple-b 1-1 } From 24ac4d2030b4d0cfc16d68f0f5a694d5c5d81ff2 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Sun, 14 Mar 2021 15:21:15 -0400 Subject: [PATCH 04/20] wip: debug: db-move These are not debug prints, don't stick them in the same commit... --- db-move | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/db-move b/db-move index 9ca39f0..93e617c 100755 --- a/db-move +++ b/db-move @@ -71,7 +71,9 @@ msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" for arch in "${ARCHES[@]}"; do declare -a add_pkgs_$arch + declare -a add_debug_pkgs_$arch declare -a remove_pkgs_$arch + declare -a remove_debug_pkgs_$arch done for pkgbase in "${args[@]:2}"; do tag_list="" @@ -94,7 +96,9 @@ for pkgbase in "${args[@]:2}"; do for tarch in "${tarches[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" + declare -n add_debug_pkgs="add_debug_pkgs_${tarch}" declare -n remove_pkgs="remove_pkgs_${tarch}" + declare -n remove_debug_pkgs="remove_debug_pkgs_${tarch}" for pkgname in "${pkgnames[@]}"; do pkgpath=$(getpkgfile "${ftppath_from}/${tarch}/${pkgname}-${pkgver}-${pkgarch}"${PKGEXTS}) pkgfile="${pkgpath##*/}" From 25f14c24934b25eced54f012a5347b6cc4c90c25 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:33:11 +0200 Subject: [PATCH 05/20] db-functions: Create DEBUGREPOS and DEBUGPKGPOOL Signed-off-by: Morten Linderud --- config | 2 ++ db-functions | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/config b/config index 2167989..13c4ed5 100644 --- a/config +++ b/config @@ -4,8 +4,10 @@ FTP_BASE="/srv/ftp" ARCHIVE_BASE="/srv/archive" ARCHIVEUSER='archive' PKGREPOS=() +DEBUGREPOS=() PKGPOOL='' SRCPOOL='' +DEBUGPKGPOOL='' STAGING_REPOS=() TESTING_REPOS=() STABLE_REPOS=() diff --git a/db-functions b/db-functions index 2751543..0274d74 100644 --- a/db-functions +++ b/db-functions @@ -443,7 +443,7 @@ check_repo_permission() { (( ${#PKGREPOS[@]} == 0 )) && return 1 [[ -z "${PKGPOOL}" ]] && return 1 - in_array "${repo}" "${PKGREPOS[@]}" || return 1 + in_array "${repo}" "${PKGREPOS[@]}" "${DEBUGREPOS[@]}" || return 1 [[ -w $FTP_BASE/${PKGPOOL} ]] || return 1 From 46fa8e318d79eb673d2a7d85b3e130f07b5713f7 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:35:03 +0200 Subject: [PATCH 06/20] test/lib: Ensure we can create debug packages, create debug pkg pool Signed-off-by: Morten Linderud --- test/Dockerfile | 2 +- test/lib/common.bash | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index 190f265..7353eb8 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM docker.io/archlinux/archlinux -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester diff --git a/test/lib/common.bash b/test/lib/common.bash index 44feb4b..7e33430 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -103,7 +103,9 @@ setup() { ARCHIVEUSER="" SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=('core' 'extra' 'testing' 'staging') + DEBUGREPOS=('core-debug' 'extra-debug' 'testing-debug' 'staging-debug') PKGPOOL='pool/packages' + DEBUGPKGPOOL='pool/packages-debug' SRCPOOL='sources/packages' STAGING_REPOS=('staging') TESTING_REPOS=('testing') @@ -128,6 +130,7 @@ eot done mkdir -p "${TMP}/ftp/${PKGPOOL}" mkdir -p "${TMP}/ftp/${SRCPOOL}" + mkdir -p "${TMP}/ftp/${DEBUGPKGPOOL}" # make dummy packages for "reproducibility" pacman -Qi | awk -F': ' '\ From 30a4c4972b8f5381c4e82b114bf1b3c804f4d164 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:40:43 +0200 Subject: [PATCH 07/20] test/lib/common: make checkPackageDB DEBUGPKGPOOL aware Signed-off-by: Morten Linderud --- test/lib/common.bash | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 7e33430..c6afdf7 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -200,7 +200,13 @@ checkPackageDB() { local pkgname local pkgarches=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${arch[@]})) - local pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + # TODO: We need a better way to figure out when we are dealing with + # debug packages + if [[ "${repo}" = *-debug ]]; then + local pkgnames=("${pkgbase}-debug") + else + local pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + fi if [[ ${pkgarches[@]} == any ]]; then repoarches=(${ARCHES[@]}) @@ -211,9 +217,8 @@ checkPackageDB() { for pkgarch in ${pkgarches[@]}; do for pkgname in ${pkgnames[@]}; do pkgfile="${pkgname}-${pkgver}-${pkgarch}${PKGEXT}" - - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] - [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] || [ -r ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgfile} ] + [ -r ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] || [ -r ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgfile}.sig ] [ ! -r ${STAGING}/${repo}/${pkgfile} ] [ ! -r ${STAGING}/${repo}/${pkgfile}.sig ] @@ -226,10 +231,12 @@ checkPackageDB() { fi [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile} ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${PKGPOOL}/${pkgfile} ] || \ + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile})" == ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgfile} ] [ -L ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig ] - [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${PKGPOOL}/${pkgfile}.sig ] || \ + [ "$(readlink -e ${FTP_BASE}/${repo}/os/${repoarch}/${pkgfile}.sig)" == ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgfile}.sig ] for db in ${DBEXT} ${FILESEXT}; do [ -r "${FTP_BASE}/${repo}/os/${repoarch}/${repo}${db%.tar.*}" ] From 3f58561a892739d83f41b2a6ea129c3d137521bc Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:43:21 +0200 Subject: [PATCH 08/20] test/lib/common: Make checkPackageDB aware of debug packages We simply remove the -debug prefix if present and ensure we are checking the SVN repository for the PKGBUILD in the actual repository. Signed-off-by: Morten Linderud --- test/lib/common.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index c6afdf7..15ac0e9 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -256,7 +256,7 @@ checkPackage() { local dirarches=() pkgbuildarches=() local pkgbuild dirarch pkgbuildver - for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; do + for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; do [[ -e $pkgbuild ]] || continue dirarch=${pkgbuild%/PKGBUILD} dirarch=${dirarch##*-} From 77098707ed23e9d7018a779a354404880f98e149 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:44:45 +0200 Subject: [PATCH 09/20] test/cases/db-update: Add debug package test Signed-off-by: Morten Linderud --- test/cases/db-update.bats | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index ddeb406..2aec4ee 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -23,6 +23,13 @@ load ../lib/common checkPackage extra 'pkg-single-arch' 1-1 } +@test "add debug package" { + releasePackage extra 'pkg-debuginfo' + db-update + checkPackage extra 'pkg-debuginfo' 1-1 + checkPackage extra-debug 'pkg-debuginfo' 1-1 +} + @test "add single epoch package" { releasePackage extra 'pkg-single-epoch' db-update From 8b540933c038dadf9746f02622e1ebb40b4c5267 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:45:46 +0200 Subject: [PATCH 10/20] db-move: Add support for debug packages The gist of this change is to always peak into the DEBUGPKGPOOL and ensure if a ${pkgbase}-debug exists in the debug repository. Signed-off-by: Morten Linderud --- db-move | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/db-move b/db-move index 93e617c..1fa08e7 100755 --- a/db-move +++ b/db-move @@ -12,7 +12,9 @@ args=("${@}") repo_from="${args[0]}" repo_to="${args[1]}" ftppath_from="${FTP_BASE}/${repo_from}/os/" +ftppath_debug_from="${FTP_BASE}/${repo_from}-debug/os/" ftppath_to="${FTP_BASE}/${repo_to}/os/" +ftppath_debug_to="${FTP_BASE}/${repo_to}-debug/os/" check_leapfrog=false if in_array "${repo_from}" "${STAGING_REPOS[@]}" && in_array "${repo_to}" "${STABLE_REPOS[@]}"; then @@ -110,6 +112,18 @@ for pkgbase in "${args[@]:2}"; do add_pkgs+=("${FTP_BASE}/${PKGPOOL}/${pkgfile}") remove_pkgs+=("${pkgname}") done + + debug_pkgpath=$(getpkgfile "${ftppath_debug_from}/${tarch}/${pkgbase}-debug-${pkgver}-${pkgarch}"${PKGEXTS}) + debug_pkgfile="${debug_pkgpath##*/}" + if [[ -f ${debug_pkgpath} ]]; then + msg2 "Found debug package %s" "${pkgbase}-debug" + ln -s "../../../${DEBUGPKGPOOL}/${debug_pkgfile}" "${ftppath_debug_to}/${tarch}/" + if [[ -f ${FTP_BASE}/${DEBUGPKGPOOL}/${debug_pkgfile}.sig ]]; then + ln -s "../../../${DEBUGPKGPOOL}/${debug_pkgfile}.sig" "${ftppath_debug_to}/${tarch}/" + fi + add_debug_pkgs+=("${FTP_BASE}/${DEBUGPKGPOOL}/${debug_pkgfile}") + remove_debug_pkgs+=("${pkgbase}-debug") + fi done fi done @@ -119,11 +133,17 @@ done for tarch in "${ARCHES[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" + declare -n add_debug_pkgs="add_debug_pkgs_${tarch}" declare -n remove_pkgs="remove_pkgs_${tarch}" + declare -n remove_debug_pkgs="remove_debug_pkgs_${tarch}" if [[ -n ${add_pkgs[@]} ]]; then arch_repo_modify add "${repo_to}" "${tarch}" "${add_pkgs[@]}" arch_repo_modify remove "${repo_from}" "${tarch}" "${remove_pkgs[@]}" fi + if [[ -n ${add_debug_pkgs[@]} ]]; then + arch_repo_modify add "${repo_to}-debug" "${tarch}" "${add_debug_pkgs[@]}" + arch_repo_modify remove "${repo_from}-debug" "${tarch}" "${remove_debug_pkgs[@]}" + fi done for pkgarch in "${ARCHES[@]}"; do From afb746e403dee67cf613e932c9ad82f7b3f5e4e8 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 20:49:56 +0200 Subject: [PATCH 11/20] test/lib/common: Make checkRemovedPackage aware of debug packages This ensures we do not have the debug repository at hand when looking at the SVN state files. However, there is no good way to figure out if we have a debug package as .BUILDINFO nor .PKGINFO has the information we need. We blindly assume that any package where we look up a *-debug repository is a debug package. Signed-off-by: Morten Linderud --- test/lib/common.bash | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 15ac0e9..c628bb2 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -283,8 +283,7 @@ checkRemovedPackage() { local pkgbase=$2 svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - - if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-"+([^-])"/PKGBUILD"; then + if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; then return 1 fi @@ -303,7 +302,14 @@ checkRemovedPackageDB() { local pkgname pkgarches=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${arch[@]})) - pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo ${pkgname[@]})) + + # TODO: We need a better way to figure out when we are dealing with + # debug packages + if [[ "${repo}" = *-debug ]]; then + pkgnames=("${pkgbase}-debug") + else + pkgnames=($(. "fixtures/$pkgbase/PKGBUILD"; echo "${pkgname[@]}")) + fi if [[ ${pkgarches[@]} == any ]]; then tarches=(${ARCHES[@]}) From 5cd41fde085f9928c6ba31c5d129fd3b5220df37 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:00:37 +0200 Subject: [PATCH 12/20] test/cases/db-move: Create a test for db-move Signed-off-by: Morten Linderud --- test/cases/db-move.bats | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 56a4331..8df0795 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -25,6 +25,39 @@ load ../lib/common checkPackage testing pkg-simple-b 1-1 } +@test "move debug package" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + echo "releasing to testing: $pkgbase" + releasePackage testing ${pkgbase} + done + + echo "db-updating..." + db-update + + echo "db-move testing -> extra pkg-debuginfo" + db-move testing extra pkg-debuginfo + + echo checkRemovedPackage testing pkg-debuginfo + checkRemovedPackage testing pkg-debuginfo + + echo checkRemovedPackage testing-debug pkg-debuginfo-debug + checkRemovedPackage testing-debug pkg-debuginfo + + echo checkPackage extra pkg-debuginfo 1-1 + checkPackage extra pkg-debuginfo 1-1 + + echo checkPackage extra-debug pkg-debuginfo 1-1 + checkPackage extra-debug pkg-debuginfo 1-1 + + echo checkPackage testing pkg-simple-b 1-1 + checkPackage testing pkg-simple-b 1-1 +} + @test "move multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') From a8fca091fe266e4a683e161052228c0ae2604aab Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:01:50 +0200 Subject: [PATCH 13/20] test/cases/testing2x: Added debug package test case Internally this uses db-move, so this should just do the correct thing. Signed-off-by: Morten Linderud --- test/cases/testing2x.bats | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/cases/testing2x.bats b/test/cases/testing2x.bats index 6127cf9..c0db45c 100644 --- a/test/cases/testing2x.bats +++ b/test/cases/testing2x.bats @@ -14,3 +14,21 @@ load ../lib/common checkPackage core pkg-any-a 1-2 checkRemovedPackage testing pkg-any-a } + + +@test "move debug package" { + releasePackage core pkg-debuginfo + db-update + + updatePackage pkg-debuginfo + + releasePackage testing pkg-debuginfo + db-update + + testing2x pkg-debuginfo + + checkPackage core pkg-debuginfo 1-2 + checkRemovedPackage testing pkg-debuginfo + checkPackage core-debug pkg-debuginfo 1-2 + checkRemovedPackage testing-debug pkg-debuginfo +} From b090f095a251350d8f20b8882c535299172e4172 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:04:34 +0200 Subject: [PATCH 14/20] db-remove: Add debug package support We check if there is a globfile match for a debug package when removing the package from the main repository. This also adds a guard to the usual package removal, and removing an unused variable. Signed-off-by: Morten Linderud --- db-remove | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/db-remove b/db-remove index a27a9f2..7ca292e 100755 --- a/db-remove +++ b/db-remove @@ -12,7 +12,6 @@ repo="$1" arch="$2" pkgbases=("${@:3}") -ftppath="$FTP_BASE/$repo/os" vcsrepo="$repo-$arch" if ! check_repo_permission "$repo"; then @@ -30,6 +29,7 @@ for tarch in "${tarches[@]}"; do done remove_pkgs=() +remove_debug_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" @@ -42,9 +42,20 @@ for pkgbase in "${pkgbases[@]}"; do warning "If it was a split package you have to remove the others yourself!" remove_pkgs+=("$pkgbase") fi + if is_globfile "${FTP_BASE}/${repo}-debug/os/${tarch}/${pkgbase}-debug"*; then + msg "Found debug package. Removing %s from [%s]..." "${pkgbase}-debug" "${repo}-debug" + remove_debug_pkgs+=("${pkgbase}-debug") + fi done for tarch in "${tarches[@]}"; do - arch_repo_modify remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + if (( ${#remove_pkgs[@]} >= 1 )); then + arch_repo_modify remove "${repo}" "${tarch}" "${remove_pkgs[@]}" + fi + + if (( ${#remove_debug_pkgs[@]} >= 1 )); then + arch_repo_modify remove "${repo}-debug" "${tarch}" "${remove_debug_pkgs[@]}" + fi + repo_unlock "$repo" "$tarch" done From 1f251d6f15ccb6a0dbc478a2fbc3d67ec5621b17 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:06:05 +0200 Subject: [PATCH 15/20] test/cases/db-remove: Added test for debug packages Signed-off-by: Morten Linderud --- test/cases/db-remove.bats | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 0597b4d..d967f6d 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -23,6 +23,31 @@ load ../lib/common done } +@test "remove debug package" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local debug_pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-remove extra ${arch} ${pkgbase} + done + done + + checkRemovedPackage extra pkg-debuginfo + for pkgbase in ${debug_pkgs[@]}; do + checkRemovedPackage extra-debug ${pkgbase} + done +} + @test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-split-a' 'pkg-split-b' 'pkg-simple-epoch') From 59a21a416d0c7d7ee23c8eb19c43b63a0af3af2e Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:06:26 +0200 Subject: [PATCH 16/20] test/cases/db-repo-add: Added test case for db-repo-add We check if there is a debug package found in the mirroring debug repository and mark this as `is_debug` if found. This creates the needed symlinks for the package pool. Signed-off-by: Morten Linderud --- test/cases/db-repo-add.bats | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test/cases/db-repo-add.bats b/test/cases/db-repo-add.bats index e49c2fa..b8f459d 100644 --- a/test/cases/db-repo-add.bats +++ b/test/cases/db-repo-add.bats @@ -6,6 +6,7 @@ __movePackageToRepo() { local arch=$3 local tarch local tarches + local is_debug=0 if [[ $arch == any ]]; then tarches=(${ARCHES[@]}) @@ -14,10 +15,18 @@ __movePackageToRepo() { fi # FIXME: pkgbase might not be part of the package filename + if __isGlobfile "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}; then + mv -v "${STAGING}"/${repo}/${pkgbase}-debug-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${DEBUGPKGPOOL}/" + is_debug=1 + fi mv -v "${STAGING}"/${repo}/${pkgbase}-*-*-${arch}${PKGEXT}{,.sig} "${FTP_BASE}/${PKGPOOL}/" for tarch in ${tarches[@]}; do ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}/os/${tarch}/" ln -sv ${FTP_BASE}/${PKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}/os/${tarch}/" + if ((is_debug)); then + ln -sv ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT} "${FTP_BASE}/${repo}-debug/os/${tarch}/" + ln -sv ${FTP_BASE}/${DEBUGPKGPOOL}/${pkgbase}-*-*-${arch}${PKGEXT}.sig "${FTP_BASE}/${repo}-debug/os/${tarch}/" + fi done } @@ -40,6 +49,25 @@ __movePackageToRepo() { done } +@test "add debug packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + for arch in ${arches[@]}; do + __movePackageToRepo extra ${pkgbase} ${arch} + db-repo-add extra ${arch} ${pkgbase}-1-1-${arch}${PKGEXT} + db-repo-add extra-debug ${arch} ${pkgbase}-debug-1-1-${arch}${PKGEXT} + done + done + + checkPackageDB extra ${pkgbase} 1-1 + checkPackageDB extra-debug ${pkgbase} 1-1 +} + @test "add multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b') From b876701fb7bfbe1d91815883d83a74c74e47f5a6 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:09:08 +0200 Subject: [PATCH 17/20] test/cases/db-repo-remove: Added debug package test This adds a test case for debug packages. It should be noted that nothing has really been changed to support this. This just check that db-repo-remove can remove a package from a *-debug repository with a -debug suffix. However, nice to have this in case regressions. Signed-off-by: Morten Linderud --- test/cases/db-repo-remove.bats | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/cases/db-repo-remove.bats b/test/cases/db-repo-remove.bats index 77bcc24..262c4f1 100644 --- a/test/cases/db-repo-remove.bats +++ b/test/cases/db-repo-remove.bats @@ -23,6 +23,30 @@ load ../lib/common done } + +@test "remove debug packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + + db-update + + for pkgbase in ${pkgs[@]}; do + for arch in ${arches[@]}; do + db-repo-remove extra ${arch} ${pkgbase} + db-repo-remove extra-debug ${arch} ${pkgbase}-debug + done + done + + checkRemovedPackageDB extra ${pkgbase} + checkRemovedPackageDB extra-debug ${pkgbase}-debug +} + @test "remove multiple packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-a' 'pkg-simple-b' 'pkg-simple-epoch') From 34e0f36d99271285e72ce72509e083605ca5ec0a Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:13:23 +0200 Subject: [PATCH 18/20] test/fixtures: add pkg-debuginfo-{a,b} This allows us to test multiple packages, which is relevant for ftpdir-cleanup later on. Signed-off-by: Morten Linderud --- test/fixtures/pkg-debuginfo-a/PKGBUILD | 13 +++++++++++++ test/fixtures/pkg-debuginfo-b/PKGBUILD | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 test/fixtures/pkg-debuginfo-a/PKGBUILD create mode 100644 test/fixtures/pkg-debuginfo-b/PKGBUILD diff --git a/test/fixtures/pkg-debuginfo-a/PKGBUILD b/test/fixtures/pkg-debuginfo-a/PKGBUILD new file mode 100644 index 0000000..b762ba1 --- /dev/null +++ b/test/fixtures/pkg-debuginfo-a/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-debuginfo-a +pkgver=1 +pkgrel=1 +pkgdesc="A package named ${pkgname}" +arch=('i686' 'x86_64') + +# guarantee that split debug packages are turned on +options=('strip' 'debug') + +package() { + install -D /dev/null "${pkgdir}"/usr/bin/hello-1 + install -D /dev/null "${pkgdir}"/../${pkgname}-debug/usr/lib/debug/usr/bin/hello-1.debug +} diff --git a/test/fixtures/pkg-debuginfo-b/PKGBUILD b/test/fixtures/pkg-debuginfo-b/PKGBUILD new file mode 100644 index 0000000..6f120c1 --- /dev/null +++ b/test/fixtures/pkg-debuginfo-b/PKGBUILD @@ -0,0 +1,13 @@ +pkgname=pkg-debuginfo-b +pkgver=1 +pkgrel=1 +pkgdesc="A package named ${pkgname}" +arch=('i686' 'x86_64') + +# guarantee that split debug packages are turned on +options=('strip' 'debug') + +package() { + install -D /dev/null "${pkgdir}"/usr/bin/hello-2 + install -D /dev/null "${pkgdir}"/../${pkgname}-debug/usr/lib/debug/usr/bin/hello-2.debug +} From f2f6880e1381a3cdd84d8e2391414148cb339104 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:14:12 +0200 Subject: [PATCH 19/20] cronjobs/ftpdir-cleanup: Support DEBUGPKGPOOL This implementation is probably the easiest one and/or cheap. After building a list of all packages in the repositories we filter the lists into two separate working lists. One for normal packages and one for debug packages. The only issue is that we can encounter globs with no matching cleanup lists, like when running ftpdir-cleanup after the debug package changes has been introduced, but before deploying new repositories. We guard against this by ensuring we have found debug packages, and normal packages, before checking the working lists. This introduces some duplicated code but should be fine. Rest of the logic is mirrored between the two package pools. Signed-off-by: Morten Linderud --- cron-jobs/ftpdir-cleanup | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/cron-jobs/ftpdir-cleanup b/cron-jobs/ftpdir-cleanup index 9df5f99..df48010 100755 --- a/cron-jobs/ftpdir-cleanup +++ b/cron-jobs/ftpdir-cleanup @@ -32,7 +32,7 @@ done [[ $CLEANUP_DRYRUN = true ]] && warning 'dry run mode is active' -for repo in "${PKGREPOS[@]}"; do +for repo in "${PKGREPOS[@]}" "${DEBUGREPOS[@]}"; do for arch in "${ARCHES[@]}"; do if [[ ! -f ${FTP_BASE}/${repo}/os/${arch}/${repo}${DBEXT} ]]; then continue @@ -66,11 +66,26 @@ for repo in "${PKGREPOS[@]}"; do done # get a list of all available packages in the package pool +HAS_POOL=0 for f in "$FTP_BASE/${PKGPOOL}"/*${PKGEXTS}; do printf '%s\n' "${f##*/}" done | sort > "${WORKDIR}/pool" + # create a list of packages in our db -cat "${WORKDIR}"/db-* 2>/dev/null | sort -u > "${WORKDIR}/db" +if [[ -s ${WORKDIR}/pool ]]; then + cat "${WORKDIR}"/db-!(*-debug-*) 2>/dev/null | sort -u > "${WORKDIR}/db" +fi + +# seperate list for debug packages +HAS_DEBUGPOOL=0 +for f in "$FTP_BASE/${DEBUGPKGPOOL}"/*${PKGEXTS}; do + printf '%s\n' "${f##*/}" +done | sort > "${WORKDIR}/debugpool" + +# seperate list for packages in our debug db +if [[ -s ${WORKDIR}/debugpool ]]; then + cat "${WORKDIR}"/db-*-debug-* 2>/dev/null | sort -u > "${WORKDIR}/debug-db" +fi old_pkgs=($(comm -23 "${WORKDIR}/pool" "${WORKDIR}/db")) if (( ${#old_pkgs[@]} >= 1 )); then @@ -81,7 +96,17 @@ if (( ${#old_pkgs[@]} >= 1 )); then done fi +old_debug_pkgs=($(comm -23 "${WORKDIR}/debugpool" "${WORKDIR}/debug-db")) +if (( ${#old_debug_pkgs[@]} >= 1 )); then + msg "Removing old packages from debug package pool..." + for old_debug_pkg in "${old_debug_pkgs[@]}"; do + msg2 '%s' "${old_debug_pkg}" + clean_pkg "$FTP_BASE/${DEBUGPKGPOOL}/${old_debug_pkg}" + done +fi + unset old_pkgs +unset old_debug_pkgs touch -d "${CLEANUP_KEEP} days ago" "${WORKDIR}/cleanup_timestamp" for f in "${CLEANUP_DESTDIR}"/**/*${PKGEXTS}; do if [[ ${WORKDIR}/cleanup_timestamp -nt $f ]]; then From 15abf5093cd15e86438a21415673bfadf586b698 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 30 Mar 2021 21:17:49 +0200 Subject: [PATCH 20/20] test/cases/ftpdir-cleanup: Add debug package test case We expand the internal __checkRepoRemovedPackage function with globfile lookups towards DEBUGPKGPOOL and the debug repository. This ensures we should get a hit for the relevant debug packages. Signed-off-by: Morten Linderud --- test/cases/ftpdir-cleanup.bats | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/cases/ftpdir-cleanup.bats b/test/cases/ftpdir-cleanup.bats index de2d2d9..7fb976c 100644 --- a/test/cases/ftpdir-cleanup.bats +++ b/test/cases/ftpdir-cleanup.bats @@ -16,9 +16,15 @@ __checkRepoRemovedPackage() { if __isGlobfile "${FTP_BASE}/${PKGPOOL}/${pkgname}"-*"${PKGEXT}"; then return 1 fi + if __isGlobfile "${FTP_BASE}/${DEBUGPKGPOOL}/${pkgname}"-debug-*"${PKGEXT}"; then + return 1 + fi if __isGlobfile "${FTP_BASE}/${repo}/os/${repoarch}/${pkgname}"-*"${PKGEXT}"; then return 1 fi + if __isGlobfile "${FTP_BASE}/${repo}-debug/os/${repoarch}/${pkgname}"-debug-*"${PKGEXT}"; then + return 1 + fi done } @@ -48,6 +54,33 @@ __checkRepoRemovedPackage() { checkPackage extra pkg-simple-b 1-1 } +@test "cleanup debug packages" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-debuginfo-a' 'pkg-debuginfo-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage extra ${pkgbase} + done + db-update + + for arch in ${arches[@]}; do + db-remove extra ${arch} "${pkgs[0]}" + done + + ftpdir-cleanup + + checkRemovedPackage extra "${pkgs[0]}" + checkRemovedPackage extra-debug "${pkgs[0]}" + for arch in ${arches[@]}; do + __checkRepoRemovedPackage extra "${pkgs[0]}" ${arch} + done + + checkPackage extra "${pkgs[1]}" 1-1 + checkPackage extra-debug "${pkgs[1]}" 1-1 +} + @test "cleanup epoch packages" { local arches=('i686' 'x86_64') local pkgs=('pkg-simple-epoch')