From 3c6cd4a6e53152879a862dbe365bab7c5a6ef40e Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Wed, 27 May 2020 00:24:35 +0200 Subject: [PATCH 01/44] basic git support using a package-repo state repository Signed-off-by: Morten Linderud --- db-functions | 6 ++-- db-functions-git | 88 ++++++++++++++++++++++++++++++++++++++++++++++++ db-move | 7 ++-- db-remove | 4 ++- db-update | 20 +++++++++++ 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 db-functions-git diff --git a/db-functions b/db-functions index 35f080c..089cdc9 100644 --- a/db-functions +++ b/db-functions @@ -357,7 +357,7 @@ check_pkgvcs() { in_array "${repo}" "${PKGREPOS[@]}" || return 1 local vcsver vcsnames=() - read -rd'\n' vcsver vcsnames < <(source_pkgbuild "${_pkgbase}" "repos/${repo}-${_pkgarch}"; \ + read -rd'\n' vcsver vcsnames < <(source_pkgbuild "${_pkgbase}" "${_pkgver}"; \ get_full_version; echo "${pkgname[@]}") read -ra vcsnames <<<"${vcsnames}" @@ -382,7 +382,8 @@ check_splitpkgs() { local _pkgbase="$(getpkgbase "${pkgfile}")" local _pkgname="$(getpkgname "${pkgfile}")" local _pkgarch="$(getpkgarch "${pkgfile}")" - local vcsnames=($(source_pkgbuild "${_pkgbase}" "repos/${repo}-${_pkgarch}"; echo "${pkgname[@]}")) + local _pkgver="$(getpkgver "${pkgfile}")" + local vcsnames=($(source_pkgbuild "${_pkgbase}" "${_pkgver}"; echo "${pkgname[@]}")) # not a split package (( ${#vcsnames[@]} > 1 )) || continue @@ -390,6 +391,7 @@ check_splitpkgs() { mkdir -p "${repo}/${_pkgarch}/${_pkgbase}" echo "${_pkgname}" >> "${repo}/${_pkgarch}/${_pkgbase}/staging" + printf '%s\n' "${vcsnames[@]}" >> "${repo}/${_pkgarch}/${_pkgbase}/vcs" done popd >/dev/null diff --git a/db-functions-git b/db-functions-git new file mode 100644 index 0000000..16e29d3 --- /dev/null +++ b/db-functions-git @@ -0,0 +1,88 @@ +#!/hint/bash + + + +# The SVN version should just check the path is present.... +fetch_pkgbuild() { + local pkgbase="${1}" + local tag="${2}" + git clone --branch "$tag" --depth 1 "${GITREPOS}${pkgbase}.git" "$WORKDIR/${pkgbase}" + git -C "${WORKDIR}/${pkgbase}" verify-tag "$tag" +} + + +# Source the PKGBUILD from the package's git/svn/whatever repo. +# Depending on how the VCS is used the tag might be "trunk" or "repos/$repo-$arch" +# or the full package version (epoch:pkgver-pkgrel) or any other recognized tag. +source_pkgbuild() { + local pkgbase="$1" + local tag="$2" + . <(cat "${WORKDIR}/${pkgbase}/PKGBUILD" 2>/dev/null || echo false) +} + +# Export PKGBUILD resource following the same rules as source_pkgbuild() +export_from_pkgrepo() { + local pkgbase="$1" + local tag="$2" + local src="$3" + local dest="$4" + + if [[ ! -e ${dest} ]]; then + mkdir -p "${dest}" + git -C "${WORKDIR}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" + fi +} + +# Which repo is this package in? +find_repo_for_package() { + local pkgbase=${1} + local pkgarch=${2} + local candidates=("${@:3}") + + local repos=($(git -C "${GITREPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ + grep -xFf <(printf "%s\n" "${candidates[@]/%/-${pkgarch}}" "${candidates[@]/%/-any}"))) + + if (( ${#repos[@]} > 1 )); then + die "%s is present in multiple repos (%s)" "${pkgbase}" "${repos[*]}" + fi + (( ${#repos[@]} == 1 )) || return $? + + printf '%s\n' "${repos[@]%/}" +} + +# Commit changes staged by (successive?) vcs_(re)?move_package runs. +vcs_commit() { + git -C "${GITREPO}" commit -m "${1}:" +} + + +vcs_update_package() { + local pkgbase="$1" + local tag="$2" + local dest="$3" + if [[ ! -e "$GITREPO/${dest}" ]]; then + die "Repository %s doesn't exist!" "${dest}" + fi + echo "$pkgbase $tag $(git -C "${WORKDIR}/${pkgbase}" rev-list -n 1 "$tag")" > "$GITREPO/$dest/$pkgbase" + git -C "${GITREPO}" add "$GITREPO/$dest/$pkgbase" || true + vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" || true +} + +# Write to the VCS in order to track a package moving between different pacman +# repositories. +vcs_move_package() { + local pkgbase=${1} + local vcsrepo_from=${GITREPO}/${2} + local vcsrepo_to=${GITREPO}/${3} + mkdir -p "${vcsrepo_to}" + git -C "$GITREPO" mv "$vcsrepo_from/${pkgbase}" "$vcsrepo_to/${pkgbase}" +} + +# Write to the VCS in order to track a package being deleted from a pacman +# repository. +vcs_remove_package() { + local pkgbase=${1} + local vcsrepo=${GITREPO}/${2}/${pkgbase} + + git -C "$GITREPO" rm "$vcsrepo" +} diff --git a/db-move b/db-move index 76e4f1e..2f7e5ac 100755 --- a/db-move +++ b/db-move @@ -43,10 +43,11 @@ for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}"; do if vcsrepo_from=$(find_repo_for_package "${pkgbase}" "${pkgarch}" "${repo_from}"); then #FIXME: abort if PKGBUILD not there - read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "repos/${vcsrepo_from}"; \ - get_full_version; echo "${pkgname[@]}") + _pkgver=$(cat "${GITREPO}/${vcsrepo_from}/${pkgbase}"| awk '{print $2}') + fetch_pkgbuild "${pkgbase}" "${_pkgver}" + read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" ""; \ + get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" - if (( ${#pkgnames[@]} < 1 )); then die "Could not read pkgname" fi diff --git a/db-remove b/db-remove index b855e5b..85180ba 100755 --- a/db-remove +++ b/db-remove @@ -33,7 +33,9 @@ remove_debug_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - if remove_pkgs+=($(source_pkgbuild "${pkgbase}" "repos/${vcsrepo}" && echo "${pkgname[@]}")); then + _pkgver=$(cat "${GITREPO}/${vcsrepo}/${pkgbase}"| awk '{print $2}') + fetch_pkgbuild "${pkgbase}" "${_pkgver}" + if remove_pkgs+=($(source_pkgbuild "${pkgbase}" "" && echo ${pkgname[@]})); then vcs_remove_package "${pkgbase}" "${vcsrepo}" vcs_commit "${0##*/}: $pkgbase removed by $(id -un)" else diff --git a/db-update b/db-update index 4156824..288f887 100755 --- a/db-update +++ b/db-update @@ -29,6 +29,19 @@ for repo in "${repos[@]}"; do done done + +for repo in "${repos[@]}"; do + pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) + if (( $? == 0 )); then + for pkg in "${pkgs[@]}"; do + echo "$pkg" + fetch_pkgbuild "$(getpkgbase ${pkg})" "$(getpkgver ${pkg})" + done + fi +done + + + # check if packages are valid for repo in "${repos[@]}"; do if ! check_repo_permission "${repo}"; then @@ -72,6 +85,8 @@ for repo in "${repos[@]}"; do if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" fi + # TODO: + # Add history repo information check as detailed in proposal else die "Could not read %s" "$STAGING" fi @@ -85,6 +100,8 @@ for repo in "${repos[@]}"; do debug_pkgs=() arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do + pkgbase="$(getpkgbase ${pkg})" + pkgver="$(getpkgver ${pkg})" pkgfile="${pkg##*/}" if is_debug_package "${pkg}"; then @@ -121,6 +138,9 @@ for repo in "${repos[@]}"; do if (( ${#debug_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}-debug" "${pkgarch}" "${debug_pkgs[@]}" fi + if ((REPO_MODIFIED)); then + vcs_update_package "$pkgbase" "$pkgver" "${repo}-${pkgarch}" + fi done done From 46d0749080414791fc6cb0b3e01811d307d2aee1 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Fri, 29 May 2020 00:42:08 +0200 Subject: [PATCH 02/44] db-update: Implement check_signed_tag Signed-off-by: Morten Linderud --- db-functions | 9 +++++++++ db-update | 3 +++ 2 files changed, 12 insertions(+) diff --git a/db-functions b/db-functions index 089cdc9..da8d100 100644 --- a/db-functions +++ b/db-functions @@ -548,4 +548,13 @@ check_reproducible() { done } +# TODO: Needs to verify the keyid is in the keyring +check_signed_tag(){ + local pkgbase="$(getpkgbase ${1})" + local pkgver="$(getpkgbase ${1})" + if ! git -C "${WORKDIR}/${pkgbase}" verify-tag "${pkgver}" 1>/dev/null ; then + return 1 + fi +} + . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-update b/db-update index 288f887..ea1e880 100755 --- a/db-update +++ b/db-update @@ -53,6 +53,9 @@ for repo in "${repos[@]}"; do if [[ -h ${pkg} ]]; then die "Package %s is a symbolic link" "$repo/${pkg##*/}" fi + if ! check_signed_tag "${pkg}"; then + die "Package %s does not have a signed tag matching the version" "$repo/${pkg##*/}" + fi if ! check_pkgfile "${pkg}"; then die "Package %s is not consistent with its meta data" "$repo/${pkg##*/}" fi From 110b56f47fa7aad6d45d32bfc5322fb24707b305 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Fri, 29 May 2020 00:48:03 +0200 Subject: [PATCH 03/44] db-update: check PKGBUILD against .BUILDINFO metadata --- db-functions | 13 +++++++++++++ db-update | 3 +++ 2 files changed, 16 insertions(+) diff --git a/db-functions b/db-functions index da8d100..a0807ec 100644 --- a/db-functions +++ b/db-functions @@ -557,4 +557,17 @@ check_signed_tag(){ fi } +# Verified the package .BUIDINFO PKGBUID sha256sum against the checked out version +check_pkgbuild_checksum(){ + local pkgbase="$(getpkgbase ${1})" + local pkgver="$(getpkgver ${1})" + local pkgfile_checksum="$(_grep_buildinfo "${1}" "pkgbuild_sha256sum")" + + local sum="$(sha256sum "${WORKDIR}/${pkgbase}/PKGBUILD")" + if [[ "$pkgfile_checksum" != "${sum%% *}" ]]; then + return 1 + fi +} + + . "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-functions-${VCS}" diff --git a/db-update b/db-update index ea1e880..678fb96 100755 --- a/db-update +++ b/db-update @@ -56,6 +56,9 @@ for repo in "${repos[@]}"; do if ! check_signed_tag "${pkg}"; then die "Package %s does not have a signed tag matching the version" "$repo/${pkg##*/}" fi + if ! check_pkgbuild_checksum "${pkg}"; then + die "Package %s was not built with the checked in PKGBUILD" "$repo/${pkg##*/}" + fi if ! check_pkgfile "${pkg}"; then die "Package %s is not consistent with its meta data" "$repo/${pkg##*/}" fi From ab971194d3881302d44b18a9892de2cd93cef7ed Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 30 May 2020 01:57:12 +0200 Subject: [PATCH 04/44] db-functions-git: Wrap git usage Signed-off-by: Morten Linderud --- db-functions-git | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/db-functions-git b/db-functions-git index 16e29d3..a76c87f 100644 --- a/db-functions-git +++ b/db-functions-git @@ -1,5 +1,18 @@ #!/hint/bash +if [[ -n ${GITUSER} ]]; then + setfacl -m u:"${GITUSER}":rwx "${WORKDIR}" + setfacl -m d:u:"${USER}":rwx "${WORKDIR}" + setfacl -m d:u:"${GITUSER}":rwx "${WORKDIR}" +fi + +arch_git() { + if [[ -z ${GITUSER} ]]; then + /usr/bin/git "${@}" + else + sudo -u "${GITUSER}" -- /usr/bin/git "${@}" + fi +} # The SVN version should just check the path is present.... @@ -29,7 +42,7 @@ export_from_pkgrepo() { if [[ ! -e ${dest} ]]; then mkdir -p "${dest}" - git -C "${WORKDIR}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" + arch_git -C "${WORKDIR}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" fi } @@ -39,7 +52,7 @@ find_repo_for_package() { local pkgarch=${2} local candidates=("${@:3}") - local repos=($(git -C "${GITREPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ + local repos=($(arch_git -C "${GITREPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ grep -xFf <(printf "%s\n" "${candidates[@]/%/-${pkgarch}}" "${candidates[@]/%/-any}"))) if (( ${#repos[@]} > 1 )); then @@ -52,7 +65,7 @@ find_repo_for_package() { # Commit changes staged by (successive?) vcs_(re)?move_package runs. vcs_commit() { - git -C "${GITREPO}" commit -m "${1}:" + arch_git -C "${GITREPO}" commit -m "${1}:" } @@ -64,7 +77,7 @@ vcs_update_package() { die "Repository %s doesn't exist!" "${dest}" fi echo "$pkgbase $tag $(git -C "${WORKDIR}/${pkgbase}" rev-list -n 1 "$tag")" > "$GITREPO/$dest/$pkgbase" - git -C "${GITREPO}" add "$GITREPO/$dest/$pkgbase" || true + arch_git -C "${GITREPO}" add "$GITREPO/$dest/$pkgbase" || true vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" || true } @@ -75,7 +88,7 @@ vcs_move_package() { local vcsrepo_from=${GITREPO}/${2} local vcsrepo_to=${GITREPO}/${3} mkdir -p "${vcsrepo_to}" - git -C "$GITREPO" mv "$vcsrepo_from/${pkgbase}" "$vcsrepo_to/${pkgbase}" + arch_git -C "$GITREPO" mv "$vcsrepo_from/${pkgbase}" "$vcsrepo_to/${pkgbase}" } # Write to the VCS in order to track a package being deleted from a pacman @@ -84,5 +97,9 @@ vcs_remove_package() { local pkgbase=${1} local vcsrepo=${GITREPO}/${2}/${pkgbase} - git -C "$GITREPO" rm "$vcsrepo" + arch_git -C "$GITREPO" rm "$vcsrepo" +} + +vcs_push(){ + arch_git -C "$GITREPO" push --tags origin master } From d769291f93bc3097a99d5ffd5e88f2fd41325c90 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 30 May 2020 01:57:33 +0200 Subject: [PATCH 05/44] db-function: tag verification against archlinux-keyring Signed-off-by: Morten Linderud --- db-functions | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions b/db-functions index a0807ec..ea8cd23 100644 --- a/db-functions +++ b/db-functions @@ -551,8 +551,8 @@ check_reproducible() { # TODO: Needs to verify the keyid is in the keyring check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" - local pkgver="$(getpkgbase ${1})" - if ! git -C "${WORKDIR}/${pkgbase}" verify-tag "${pkgver}" 1>/dev/null ; then + local pkgver="$(getpkgver ${1})" + if ! GNUPGHOME=/etc/pacman.d/gnupg git -C "${WORKDIR}/${pkgbase}" verify-tag "${pkgver}" 1>/dev/null ; then return 1 fi } From 7f72eaf1992f6d7e01560eb35cd1a9b7c775137d Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 30 May 2020 01:58:46 +0200 Subject: [PATCH 06/44] db-*: ensure we abort if we can't find the PKGBUILD Signed-off-by: Morten Linderud --- db-move | 7 ++++--- db-update | 5 +++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/db-move b/db-move index 2f7e5ac..5d50a7d 100755 --- a/db-move +++ b/db-move @@ -42,9 +42,10 @@ for pkgbase in "${args[@]:2}"; do found=false for pkgarch in "${ARCHES[@]}"; do if vcsrepo_from=$(find_repo_for_package "${pkgbase}" "${pkgarch}" "${repo_from}"); then - #FIXME: abort if PKGBUILD not there - _pkgver=$(cat "${GITREPO}/${vcsrepo_from}/${pkgbase}"| awk '{print $2}') - fetch_pkgbuild "${pkgbase}" "${_pkgver}" + _pkgver=$(awk '{print $2}' "${GITREPO}/${vcsrepo_from}/${pkgbase}") + if ! fetch_pkgbuild "${pkgbase}" "${_pkgver}"; then + die "Couldn't find package %s in git!" "${pkgbase}" + fi read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" ""; \ get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" diff --git a/db-update b/db-update index 678fb96..b25fdf3 100755 --- a/db-update +++ b/db-update @@ -34,8 +34,9 @@ for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) if (( $? == 0 )); then for pkg in "${pkgs[@]}"; do - echo "$pkg" - fetch_pkgbuild "$(getpkgbase ${pkg})" "$(getpkgver ${pkg})" + if ! fetch_pkgbuild "$(getpkgbase ${pkg})" "$(getpkgver ${pkg})"; then + die "Couldn't find package %s in git!" "${pkg}" + fi done fi done From c3b9b5d0a6465f2f55485d307baac8bd0cd9d60d Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 30 May 2020 19:11:13 +0200 Subject: [PATCH 07/44] db-move: Ensure we are only acting on the git repo after REPO_MODIFIED is set Signed-off-by: Morten Linderud --- db-move | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/db-move b/db-move index 5d50a7d..167fad7 100755 --- a/db-move +++ b/db-move @@ -95,7 +95,6 @@ for pkgbase in "${args[@]:2}"; do get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" - vcs_move_package "${pkgbase}" "${repo_from}-${pkgarch}" "${repo_to}-${pkgarch}" tag_list+=", $pkgarch" for tarch in "${tarches[@]}"; do @@ -130,7 +129,6 @@ for pkgbase in "${args[@]:2}"; do fi done tag_list="${tag_list#, }" - vcs_commit "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})" done for tarch in "${ARCHES[@]}"; do @@ -148,6 +146,21 @@ for tarch in "${ARCHES[@]}"; do fi done +# Only modify repository if everything else has been done +# We really don't want it to represent a broken state +if ((REPO_MODIFIED)); then + for pkgbase in "${args[@]:2}"; do + for tarch in "${ARCHES[@]}"; do + if ! vcs_move_package "${pkgbase}" "${repo_from}-${tarch}" "${repo_to}-${tarch}"; then + die "Couldn't move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" + fi + if ! vcs_commit "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})"; then + die "Couldn't commit the move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" + fi + done + done +fi + for pkgarch in "${ARCHES[@]}"; do repo_unlock "${repo_from}" "${pkgarch}" repo_unlock "${repo_to}" "${pkgarch}" From 19b512d7263011717c706b060ae7ee648971eb31 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Tue, 15 Sep 2020 23:58:22 +0200 Subject: [PATCH 08/44] git: Implement tags as in the conversion script Signed-off-by: Morten Linderud Signed-off-by: Levente Polyak --- db-functions | 2 +- db-functions-git | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/db-functions b/db-functions index ea8cd23..45a64a5 100644 --- a/db-functions +++ b/db-functions @@ -551,7 +551,7 @@ check_reproducible() { # TODO: Needs to verify the keyid is in the keyring check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" - local pkgver="$(getpkgver ${1})" + local pkgver="$(conv_tag $(getpkgver ${1}))" if ! GNUPGHOME=/etc/pacman.d/gnupg git -C "${WORKDIR}/${pkgbase}" verify-tag "${pkgver}" 1>/dev/null ; then return 1 fi diff --git a/db-functions-git b/db-functions-git index a76c87f..01ede5e 100644 --- a/db-functions-git +++ b/db-functions-git @@ -6,6 +6,16 @@ if [[ -n ${GITUSER} ]]; then setfacl -m d:u:"${GITUSER}":rwx "${WORKDIR}" fi +# Converts from the PKGBUILD tag to the git repository tag +# Input 1:1.0~0-1 +# Output 1-1.0.0-1 +conv_tag() { + local git_tag="$1" + git_tag="${git_tag/:/-}" + git_tag="${git_tag//\~/.}" + printf "%s" "${git_tag}" +} + arch_git() { if [[ -z ${GITUSER} ]]; then /usr/bin/git "${@}" @@ -18,25 +28,23 @@ arch_git() { # The SVN version should just check the path is present.... fetch_pkgbuild() { local pkgbase="${1}" - local tag="${2}" + local tag=$(conv_tag "${2}") git clone --branch "$tag" --depth 1 "${GITREPOS}${pkgbase}.git" "$WORKDIR/${pkgbase}" git -C "${WORKDIR}/${pkgbase}" verify-tag "$tag" } # Source the PKGBUILD from the package's git/svn/whatever repo. -# Depending on how the VCS is used the tag might be "trunk" or "repos/$repo-$arch" -# or the full package version (epoch:pkgver-pkgrel) or any other recognized tag. source_pkgbuild() { local pkgbase="$1" - local tag="$2" + local tag=$(conv_tag ${2}) . <(cat "${WORKDIR}/${pkgbase}/PKGBUILD" 2>/dev/null || echo false) } # Export PKGBUILD resource following the same rules as source_pkgbuild() export_from_pkgrepo() { local pkgbase="$1" - local tag="$2" + local tag=$(conv_tag ${2}) local src="$3" local dest="$4" @@ -71,12 +79,13 @@ vcs_commit() { vcs_update_package() { local pkgbase="$1" - local tag="$2" + local tag="${2}" + local gittag="$(conv_tag ${2})" local dest="$3" if [[ ! -e "$GITREPO/${dest}" ]]; then die "Repository %s doesn't exist!" "${dest}" fi - echo "$pkgbase $tag $(git -C "${WORKDIR}/${pkgbase}" rev-list -n 1 "$tag")" > "$GITREPO/$dest/$pkgbase" + echo "$pkgbase $gittag $(git -C "${WORKDIR}/${pkgbase}" rev-list -n 1 "$gittag")" > "$GITREPO/$dest/$pkgbase" arch_git -C "${GITREPO}" add "$GITREPO/$dest/$pkgbase" || true vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" || true } From df613422263742c9cce95a11ffc372d31c664ac1 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 8 May 2021 15:02:11 +0200 Subject: [PATCH 09/44] db-update: Support multiple packages in one transaction We are faking a proper hashmap because we need more information then simply the pkgfile name. Signed-off-by: Morten Linderud --- db-update | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/db-update b/db-update index b25fdf3..5069ff1 100755 --- a/db-update +++ b/db-update @@ -105,6 +105,7 @@ for repo in "${repos[@]}"; do for pkgarch in "${ARCHES[@]}"; do add_pkgs=() debug_pkgs=() + declare -A add_vcspkgs arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do pkgbase="$(getpkgbase ${pkg})" @@ -138,6 +139,9 @@ for repo in "${repos[@]}"; do if [[ -f $FTP_BASE/${currentpool}/${pkgfile}.sig ]]; then ln -sf "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi + add_pkgs+=("${pkgfile}") + add_vcspkgs["${pkgfile}::pkgbase"]="${pkgbase}" + add_vcspkgs["${pkgfile}::pkgver"]="${pkgver}" done if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" "${add_pkgs[@]}" @@ -146,7 +150,11 @@ for repo in "${repos[@]}"; do arch_repo_modify add "${repo}-debug" "${pkgarch}" "${debug_pkgs[@]}" fi if ((REPO_MODIFIED)); then - vcs_update_package "$pkgbase" "$pkgver" "${repo}-${pkgarch}" + for pkg in "${add_pkgs[@]}"; do + pkgbase="${add_vcspkgs[$pkg::pkgbase]}" + pkgver="${add_vcspkgs[$pkg::pkgver]}" + vcs_update_package "$pkgbase" "$pkgver" "${repo}-${pkgarch}" + done fi done done From eac1843b592868569d12380c11fcba14a902c569 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 8 May 2021 15:30:36 +0200 Subject: [PATCH 10/44] db-functions: Ignore clone if repo is checked out When dealing with multiple architectures the upstream repo is going to be checked out when we reach this. Ignore the clone if it does. Signed-off-by: Morten Linderud --- db-functions-git | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions-git b/db-functions-git index 01ede5e..0b3915f 100644 --- a/db-functions-git +++ b/db-functions-git @@ -29,7 +29,7 @@ arch_git() { fetch_pkgbuild() { local pkgbase="${1}" local tag=$(conv_tag "${2}") - git clone --branch "$tag" --depth 1 "${GITREPOS}${pkgbase}.git" "$WORKDIR/${pkgbase}" + [ -d "$WORKDIR/${pkgbase}" ] || git clone --branch "$tag" --depth 1 "${GITREPOS}/${pkgbase}.git" "$WORKDIR/${pkgbase}" git -C "${WORKDIR}/${pkgbase}" verify-tag "$tag" } From a1b4f0d99380258f99469ba0ca2ea74a36b77372 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 8 May 2021 15:30:08 +0200 Subject: [PATCH 11/44] test/lib: Added git repo support Signed-off-by: Morten Linderud --- test/lib/common.bash | 123 +++++++++++++++++++++++++++++++------------ 1 file changed, 88 insertions(+), 35 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index b66ec95..2003823 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -1,12 +1,24 @@ . /usr/share/makepkg/util.sh shopt -s extglob + +# Copy of db-functions-git +arch_git() { + if [[ -z ${GITUSER} ]]; then + /usr/bin/git "${@}" + else + sudo -u "${GITUSER}" -- /usr/bin/git "${@}" + fi +} + __updatePKGBUILD() { local pkgrel pkgrel=$(. PKGBUILD; expr ${pkgrel} + 1) sed "s/pkgrel=.*/pkgrel=${pkgrel}/" -i PKGBUILD - svn commit -q -m"update pkg to pkgrel=${pkgrel}" + git add . + git commit -m "update pkg to pkgrel=${pkgrel}" + git push } __getCheckSum() { @@ -15,6 +27,18 @@ __getCheckSum() { echo "${result%% *}" } +# Converts from the git repository tag to the PKGBUILD tag +# Input 1-1.0.0-1 +# Output 1:1.0.0-1 +__parseGitTag(){ + tag="${1}" + while IFS=- read -r pkgrel pkgver epoch; do + test -n "${epoch}" && printf "%s:" "$epoch" + printf "%s" "$(echo "$pkgver" | rev)" + printf "%s" "$(echo "$pkgrel-" | rev)" + done < <(echo "${tag}" | rev) +} + # Proxy function to check if a file exists. Using [[ -f ... ]] directly is not # always wanted because we might want to expand bash globs first. This way we # can pass unquoted globs to __isGlobfile() and have them expanded as function @@ -69,22 +93,10 @@ __archrelease() { local tarch local tag - pkgarches=($(. PKGBUILD; echo ${arch[@]})) - pushd .. - for tarch in ${pkgarches[@]}; do - tag=${repo}-${tarch} - - if [[ -d repos/$tag ]]; then - svn rm repos/$tag/PKGBUILD - else - mkdir -p repos/$tag - svn add repos/$tag - fi - - svn copy -r HEAD trunk/PKGBUILD repos/$tag/ - done - svn commit -m "__archrelease" - popd + pkgver=$(. PKGBUILD; get_full_version) + gittag=${pkgver/:/-} + git tag -s -m "released $pkgbase-$pkgver" "$gittag" + git push --tags origin main } setup() { @@ -95,13 +107,13 @@ setup() { PKGEXT=".pkg.tar.xz" TMP="$(mktemp -d)" + chmod 770 "$TMP" export DBSCRIPTS_CONFIG=${TMP}/config.local cat < "${DBSCRIPTS_CONFIG}" FTP_BASE="${TMP}/ftp" ARCHIVE_BASE="${TMP}/archive" ARCHIVEUSER="" - SVNREPO="file://${TMP}/svn-packages-repo" PKGREPOS=('core' 'extra' 'testing' 'staging') DEBUGREPOS=('core-debug' 'extra-debug' 'testing-debug' 'staging-debug') PKGPOOL='pool/packages' @@ -117,10 +129,28 @@ setup() { ARCHES=(x86_64 i686) CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false + VCS=git + GNUPGHOME="/etc/pacman.d/gnupg" + GITREPOS="${TMP}/git-packages" + GITREPO="${TMP}/repository" + GITUSER="git-packages" eot + + . config - mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup,svn-packages-{copy,repo}} + git config --global user.email "tester@localhost" + git config --global user.name "Bob Tester" + git config --global init.defaultBranch main + git config --global advice.detachedHead false + + + # This is for our git clones when initializing bare repos + TMP_WORKDIR_GIT=${TMP}/git-clones + + mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup} + mkdir -p "${GITREPOS}/packages" + mkdir -p "${TMP_WORKDIR_GIT}" for r in ${PKGREPOS[@]}; do mkdir -p "${TMP}"/staging/${r}{,-debug} @@ -144,8 +174,20 @@ eot touch "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}/${line}"{,.sig} done - svnadmin create "${TMP}/svn-packages-repo" - svn checkout -q "file://${TMP}/svn-packages-repo" "${TMP}/svn-packages-copy" + git init --bare --shared=group "${TMPDIR}/git-packages-bare.git" + mkdir "${GITREPO}" + chmod 777 "${GITREPO}" + arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GITREPO}" + for r in ${PKGREPOS[@]}; do + for a in ${ARCHES[@]}; do + # This is ugly but we need 770 in the test env + (umask 002; + mkdir -p "${GITREPO}/${r}-${a}"; + touch "${GITREPO}/${r}-${a}"/.gitkeep; + arch_git -C "${GITREPO}" add "${GITREPO}/${r}-${a}"/.gitkeep) + done + done + arch_git -C "${GITREPO}" commit -m "init repos" } teardown() { @@ -156,23 +198,33 @@ releasePackage() { local repo=$1 local pkgbase=$2 - if [ ! -d "${TMP}/svn-packages-copy/${pkgbase}/trunk" ]; then - mkdir -p "${TMP}/svn-packages-copy/${pkgbase}"/{trunk,repos} - cp -r "fixtures/${pkgbase}"/* "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ - svn add -q "${TMP}/svn-packages-copy"/${pkgbase} - svn commit -q -m"initial commit of ${pkgbase}" "${TMP}/svn-packages-copy" + if [ ! -d "${GITREPOS}/packages/${pkgbase}.git" ]; then + git init --bare --shared=all "${GITREPOS}/packages/${pkgbase}".git + git -c "core.sharedRepository=group" clone "${GITREPOS}/packages/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" + cp -r "fixtures/${pkgbase}"/* "${TMP_WORKDIR_GIT}/${pkgbase}" + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" add "${TMP_WORKDIR_GIT}/${pkgbase}"/* + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" commit -m "initial commit of ${pkgbase}" + git -C "${TMP_WORKDIR_GIT}/${pkgbase}" push + + fi + + if [ ! -d "${TMP_WORKDIR_GIT}/${pkgbase}" ]; then + git clone "${GITREPOS}/packages/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" fi - pushd "${TMP}/svn-packages-copy"/${pkgbase}/trunk/ + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + git pull origin main __buildPackage "${STAGING}"/${repo} __archrelease ${repo} + chmod -R 777 "${GITREPOS}/packages/" popd } updatePackage() { local pkgbase=$1 - pushd "${TMP}/svn-packages-copy/${pkgbase}/trunk/" + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + git pull origin main __updatePKGBUILD __buildPackage popd @@ -252,19 +304,20 @@ checkPackage() { local pkgbase=$2 local pkgver=$3 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - local dirarches=() pkgbuildarches=() local pkgbuild dirarch pkgbuildver - for pkgbuild in "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; do + for pkgbuild in "${GITREPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; do [[ -e $pkgbuild ]] || continue - dirarch=${pkgbuild%/PKGBUILD} + dirarch=${pkgbuild%/${pkgbase}} dirarch=${dirarch##*-} dirarches+=("$dirarch") - pkgbuildarches+=($(. "$pkgbuild"; echo ${arch[@]})) - pkgbuildver=$(. "$pkgbuild"; get_full_version) - [[ $pkgver = "$pkgbuildver" ]] + pkgbuildarches+=($(. "${TMP_WORKDIR_GIT}/${pkgbase}/PKGBUILD"; echo ${arch[@]})) + + while read -r _ tag _; do + pkgbuildver=$(__parseGitTag "$tag") + [[ $pkgver = "$pkgbuildver" ]] + done < "$pkgbuild" done # Verify that the arches-from-dirnames and # arches-from-PKGBUILDs agree (that a PKGBUILD existed for From 45a738b30303d743c3eb6fcdc91dbe50fca990c7 Mon Sep 17 00:00:00 2001 From: Morten Linderud Date: Sat, 30 May 2020 19:12:54 +0200 Subject: [PATCH 12/44] config: add git example config Signed-off-by: Morten Linderud --- config.local.git | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 config.local.git diff --git a/config.local.git b/config.local.git new file mode 100644 index 0000000..2a5dbb1 --- /dev/null +++ b/config.local.git @@ -0,0 +1,31 @@ +#!/hint/bash + +VCS=git + +PKGREPOS=( + core core-staging core-testing + extra extra-staging extra-testing + multilib multilib-staging multilib-testing + kde-unstable gnome-unstable +) +DEBUGREPOS=( + core-debug core-staging-debug core-testing-debug + extra-debug extra-staging-debug extra-testing-debug + multilib-debug multilib-staging-debug multilib-testing-debug + kde-unstable-debug gnome-unstable-debug +) +TESTING_REPOS=(core-testing extra-testing multilib-testing) +STABLE_REPOS=(core extra) + +PKGPOOL='pool/packages' +DEBUGPKGPOOL='pool/packages-debug' +SRCPOOL='sources/packages' + +CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" +SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" +TMPDIR="/srv/repos/svn-packages/tmp" + +export GNUPGHOME="/etc/pacman.d/gnupg" +GITREPOS="https://gitlab.archlinux.org/archlinux/packaging/packages" +GITREPO="/repository" +GITUSER="git" From 5e7225302461f41e4298a7028331b0c9e79b2609 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 12 Jul 2022 21:40:49 +0200 Subject: [PATCH 13/44] test: port check if package is removed to git --- test/lib/common.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 2003823..a337c7a 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -335,8 +335,7 @@ checkRemovedPackage() { local repo=$1 local pkgbase=$2 - svn up -q "${TMP}/svn-packages-copy/${pkgbase}" - if __isGlobfile "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo%-debug}-"+([^-])"/PKGBUILD"; then + if __isGlobfile "${GITREPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; then return 1 fi From 13bd810dee9e05f9559dea6c41ad92d58d576c6b Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 12 Jul 2022 22:42:27 +0200 Subject: [PATCH 14/44] test: fix helper function to update repo PKGBUILDs --- test/lib/common.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index a337c7a..a21c867 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -235,7 +235,7 @@ updateRepoPKGBUILD() { local repo=$2 local arch=$3 - pushd "${TMP}/svn-packages-copy/${pkgbase}/repos/${repo}-${arch}/" + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" __updatePKGBUILD popd } From 80d1f025cde4d8fe32246edf29853b3b432cd046 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 15 Jul 2022 21:03:56 +0200 Subject: [PATCH 15/44] test: fix release helper by avoiding duplicate tags on the same commit The way a lot of test cases are build collides with the way our git packaging works. We tag every release in the API, which will fail for some of those tests setups as the tag already exists. Fix this issue by only trying to tag a version if the revision we try to tag actually differs from an existing tag. This allows to preserve the easy going test cases as the functional ideology of the test case won't be compromised by this test check. --- test/lib/common.bash | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/lib/common.bash b/test/lib/common.bash index a21c867..819ef34 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -92,9 +92,22 @@ __archrelease() { local pkgarches local tarch local tag + local rev + local head pkgver=$(. PKGBUILD; get_full_version) gittag=${pkgver/:/-} + + # avoid trying to tag the same commit twice + if rev=$(git rev-list -n1 "$gittag" 2>/dev/null); then + head=$(git rev-parse HEAD) + if [[ "$rev" != "$head" ]]; then + error "failed to tag revision %s" "${head}" + error "tag '%s' already exists for revision %s" "${gittag}" "${rev}" + exit 1 + fi + return 0 + fi git tag -s -m "released $pkgbase-$pkgver" "$gittag" git push --tags origin main } From 1ba72ba5fcea66e14ce5a42b4b9c815e062589ce Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 25 Jul 2022 22:02:54 +0200 Subject: [PATCH 16/44] db-update: fix retrieving metadata from any-packages for multi arch For any-packages on a multi arch repo setup, we can't call `getpkgbase` or `getpkgver` on the original `$pkg` location, as they get moved to a different location during per arch repo target handling. Instead of trying to refactor all the behavior of doing so, lets instead choose to retrieve this information once as first action and only access the cached metadata afterwards. Before this commit, the cache has only been populated as last action of the update process, which will fail horribly for the second set of packages targeting another architecture's repo. --- db-update | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/db-update b/db-update index 5069ff1..5406d72 100755 --- a/db-update +++ b/db-update @@ -101,16 +101,22 @@ done for repo in "${repos[@]}"; do msg "Updating [%s]..." "$repo" + declare -A add_vcspkgs any_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*-any${PKGEXTS} 2>/dev/null)) for pkgarch in "${ARCHES[@]}"; do add_pkgs=() debug_pkgs=() - declare -A add_vcspkgs arch_pkgs=($(getpkgfiles "${STAGING}/${repo}/"*"-${pkgarch}"${PKGEXTS} 2>/dev/null)) for pkg in "${arch_pkgs[@]}" "${any_pkgs[@]}"; do - pkgbase="$(getpkgbase ${pkg})" - pkgver="$(getpkgver ${pkg})" pkgfile="${pkg##*/}" + if [[ ! "${add_vcspkgs["${pkgfile}::pkgbase"]+exists}" ]]; then + if ! pkgbase="$(getpkgbase "${pkg}")" || \ + ! pkgver="$(getpkgver "${pkg}")"; then + die "Failed to get metadata from '%s'" "${pkg}" + fi + add_vcspkgs["${pkgfile}::pkgbase"]="${pkgbase}" + add_vcspkgs["${pkgfile}::pkgver"]="${pkgver}" + fi if is_debug_package "${pkg}"; then debug_pkgs+=("${pkgfile}") @@ -139,9 +145,6 @@ for repo in "${repos[@]}"; do if [[ -f $FTP_BASE/${currentpool}/${pkgfile}.sig ]]; then ln -sf "../../../${currentpool}/${pkgfile}.sig" "$FTP_BASE/${currentrepo}/os/${pkgarch}" fi - add_pkgs+=("${pkgfile}") - add_vcspkgs["${pkgfile}::pkgbase"]="${pkgbase}" - add_vcspkgs["${pkgfile}::pkgver"]="${pkgver}" done if (( ${#add_pkgs[@]} >= 1 )); then arch_repo_modify add "${repo}" "${pkgarch}" "${add_pkgs[@]}" From 9c7a429517220e65cac7b289bef33c3d4d3c98f7 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 16 Aug 2022 01:05:14 +0200 Subject: [PATCH 17/44] db-update: fix is_debug_package handling for any-packages on multi arch For multi arch setups when handling any packages, we can't call `getpkgbase` or `getpkgver` on the original `$pkg` location, as they get moved along to a different location during looping through the arches. This means we only have a single shot on `any` packages to collect all the metadata that requires the actual package file at `$pkg` to exist. Before this commit, the code only worked by sheer luck, as is_debug_package will call `getpkgbase` via a subshell, which will silently fail horribly as its run in a subshell. Fix this unexpected behavior by only querying is_debug_package when the actual file still exists. Any calls to is_debug_package with a none existing path are now considered a fatal error. Related-to: db-update: fix retrieving metadata from any-packages for multi arch --- db-functions | 13 +++++++++---- db-update | 5 ++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/db-functions b/db-functions index 45a64a5..e0bd648 100644 --- a/db-functions +++ b/db-functions @@ -265,11 +265,16 @@ getpkgdesc() { # here be dragons is_debug_package() { local pkgfile=${1} - local pkgbase="$(getpkgbase "${pkgfile}")" - local pkgname="$(getpkgname "${pkgfile}")" - local pkgdesc="$(getpkgdesc "${pkgfile}")" + local pkgbase pkgname pkgdesc - [[ ${pkgdesc} == "Detached debugging symbols for "* && ${pkgbase}-debug = ${pkgname} ]] + if ! pkgbase="$(getpkgbase "${pkgfile}")" || \ + ! pkgname="$(getpkgname "${pkgfile}")" || \ + ! pkgdesc="$(getpkgdesc "${pkgfile}")"; then + error "Failed to get PKGINFO metadata from package '%s'" "${pkgfile}" + exit 1 + fi + + [[ ${pkgdesc} == "Detached debugging symbols for "* && ${pkgbase}-debug = "${pkgname}" ]] } check_packager() { diff --git a/db-update b/db-update index 5406d72..fe2a1c7 100755 --- a/db-update +++ b/db-update @@ -116,9 +116,12 @@ for repo in "${repos[@]}"; do fi add_vcspkgs["${pkgfile}::pkgbase"]="${pkgbase}" add_vcspkgs["${pkgfile}::pkgver"]="${pkgver}" + + is_debug_package "${pkg}" + add_vcspkgs["${pkgfile}::debug"]=$(( $? == 0 )) fi - if is_debug_package "${pkg}"; then + if (( ${add_vcspkgs["${pkgfile}::debug"]} )); then debug_pkgs+=("${pkgfile}") currentpool=${PKGPOOL}-debug currentrepo=${repo}-debug From 655817c526836937db0a0aa2094cc6858cabc92a Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 3 Sep 2022 22:54:56 +0200 Subject: [PATCH 18/44] adjust git packaging repos to be stored in a single location Reflect first batch of changes related to RFC-14 (Merge package repositories). https://gitlab.archlinux.org/archlinux/rfcs/-/merge_requests/14 --- test/lib/common.bash | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/lib/common.bash b/test/lib/common.bash index 819ef34..35c32ef 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -157,12 +157,12 @@ eot git config --global init.defaultBranch main git config --global advice.detachedHead false - + # This is for our git clones when initializing bare repos TMP_WORKDIR_GIT=${TMP}/git-clones mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup} - mkdir -p "${GITREPOS}/packages" + mkdir -p "${GITREPOS}" mkdir -p "${TMP_WORKDIR_GIT}" for r in ${PKGREPOS[@]}; do @@ -211,9 +211,9 @@ releasePackage() { local repo=$1 local pkgbase=$2 - if [ ! -d "${GITREPOS}/packages/${pkgbase}.git" ]; then - git init --bare --shared=all "${GITREPOS}/packages/${pkgbase}".git - git -c "core.sharedRepository=group" clone "${GITREPOS}/packages/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" + if [ ! -d "${GITREPOS}/${pkgbase}.git" ]; then + git init --bare --shared=all "${GITREPOS}/${pkgbase}".git + git -c "core.sharedRepository=group" clone "${GITREPOS}/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" cp -r "fixtures/${pkgbase}"/* "${TMP_WORKDIR_GIT}/${pkgbase}" git -C "${TMP_WORKDIR_GIT}/${pkgbase}" add "${TMP_WORKDIR_GIT}/${pkgbase}"/* git -C "${TMP_WORKDIR_GIT}/${pkgbase}" commit -m "initial commit of ${pkgbase}" @@ -222,14 +222,14 @@ releasePackage() { fi if [ ! -d "${TMP_WORKDIR_GIT}/${pkgbase}" ]; then - git clone "${GITREPOS}/packages/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" + git clone "${GITREPOS}/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" fi pushd "${TMP_WORKDIR_GIT}/${pkgbase}" git pull origin main __buildPackage "${STAGING}"/${repo} __archrelease ${repo} - chmod -R 777 "${GITREPOS}/packages/" + chmod -R 777 "${GITREPOS}/" popd } From 715185e4582fbf3f63f6f5986f6465238d3035bb Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 4 Sep 2022 00:02:08 +0200 Subject: [PATCH 19/44] implemented unix group based ACL to shield packaging repositories Introduce an associate array that can be set in the config to define the permissions per group. Declare permission sub sets so we can easier puzzle together the actual permissions per group without duplicating all of the repos in the declaration. Furthermore we specifically use `packager` and `junior-packager` instead of `tu` to reflect the rename on system group level. Reflect second batch of changes related to RFC-14 (Merge package repositories). https://gitlab.archlinux.org/archlinux/rfcs/-/merge_requests/14 --- config | 1 + config.local.git | 33 +++++++++++++++++++++++++++++ db-functions | 10 ++++++++- test/cases/db-move.bats | 44 +++++++++++++++++++++++++++++++++++++++ test/cases/db-remove.bats | 15 +++++++++++++ test/cases/db-update.bats | 13 +++++++++++- test/lib/common.bash | 21 ++++++++++++++++--- 7 files changed, 132 insertions(+), 5 deletions(-) diff --git a/config b/config index 13c4ed5..b4144d6 100644 --- a/config +++ b/config @@ -11,6 +11,7 @@ DEBUGPKGPOOL='' STAGING_REPOS=() TESTING_REPOS=() STABLE_REPOS=() +declare -A ACL=() # VCS backend VCS=svn diff --git a/config.local.git b/config.local.git index 2a5dbb1..abd0dbf 100644 --- a/config.local.git +++ b/config.local.git @@ -17,6 +17,39 @@ DEBUGREPOS=( TESTING_REPOS=(core-testing extra-testing multilib-testing) STABLE_REPOS=(core extra) +ACL_CORE_LIMITED=( + core-staging core-staging-debug + core-testing core-testing-debug +) +ACL_CORE_ALL=( + core core-debug + "${ACL_CORE_LIMITED[@]}" +) +ACL_EXTRA_LIMITED=( + extra-staging extra-staging-debug + extra-testing extra-testing-debug +) +ACL_EXTRA_ALL=( + extra extra-debug + "${ACL_EXTRA_LIMITED[@]}" +) +ACL_DESKTOP=( + kde-unstable kde-unstable-debug + gnome-unstable gnome-unstable-debug +) +ACL_MULTILIB=( + multilib multilib-debug + multilib-staging multilib-staging-debug + multilib-testing multilib-testing-debug +) +ACL=( + [packager]="${ACL_EXTRA_ALL[@]}" + [junior-packager]="${ACL_EXTRA_LIMITED[@]}" + [dev]="${ACL_CORE_ALL[@]} ${ACL_EXTRA_ALL[@]} ${ACL_DESKTOP[@]}" + [junior-dev]="${ACL_CORE_LIMITED[@]} ${ACL_EXTRA_ALL[@]} ${ACL_DESKTOP[@]}" + [multilib]="${ACL_MULTILIB[@]}" +) + PKGPOOL='pool/packages' DEBUGPKGPOOL='pool/packages-debug' SRCPOOL='sources/packages' diff --git a/db-functions b/db-functions index e0bd648..8855bc1 100644 --- a/db-functions +++ b/db-functions @@ -491,7 +491,15 @@ check_repo_permission() { [[ -f ${dir}${repo}${FILESEXT} && ! -w ${dir}${repo}${FILESEXT} ]] && return 1 done - return 0 + # Check unix group against ACL + for group in $(/usr/bin/groups); do + [ "${ACL[${group}]+exists}" ] || continue + read -r -a acl_repos <<< "${ACL[${group}]}" + in_array "${repo}" "${acl_repos[@]}" || continue + return 0 + done + + return 1 } set_repo_permission() { diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 3dd0556..64d8015 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -193,3 +193,47 @@ load ../lib/common checkPackage extra pkg-split-debuginfo 1-1 checkPackage extra-debug pkg-split-debuginfo 1-1 } + +@test "move package with insufficient target repo permissions fails" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage testing ${pkgbase} + done + + db-update + + run db-move testing noperm pkg-simple-a pkg-simple-b + [ "$status" -ne 0 ] + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage noperm ${pkgbase} + checkPackage testing ${pkgbase} 1-1 + done +} + +@test "move package with insufficient source repo permissions fails" { + local arches=('i686' 'x86_64') + local pkgs=('pkg-simple-a' 'pkg-simple-b') + local pkgbase + local arch + + for pkgbase in ${pkgs[@]}; do + releasePackage noperm ${pkgbase} + done + + enablePermission noperm + db-update + disablePermissionOverride + + run db-move noperm testing pkg-simple-a pkg-simple-b + [ "$status" -ne 0 ] + + for pkgbase in ${pkgs[@]}; do + checkRemovedPackage testing ${pkgbase} + checkPackage noperm ${pkgbase} 1-1 + done +} diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 484789a..9aadffb 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -132,3 +132,18 @@ load ../lib/common checkRemovedPackage extra ${pkgbase} done } + +@test "remove package with insufficient repo permissions fails" { + local pkgbase='pkg-any-a' + + releasePackage noperm ${pkgbase} + + enablePermission noperm + db-update + disablePermissionOverride + + run db-remove noperm any ${pkgbase} + [ "$status" -ne 0 ] + + checkPackage noperm ${pkgbase} 1-1 +} diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 8d0851e..00f86b9 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -255,7 +255,7 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-a' } -@test "add package with insufficient permissions fails" { +@test "add package with insufficient directory permissions fails" { releasePackage core 'pkg-any-a' releasePackage extra 'pkg-any-b' @@ -268,6 +268,17 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-b' } +@test "add package with insufficient repo permissions fails" { + releasePackage noperm 'pkg-any-a' + releasePackage extra 'pkg-any-b' + + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackageDB noperm 'pkg-any-a' + checkRemovedPackageDB extra 'pkg-any-b' +} + @test "package has to be aregular file" { local p local target=$(mktemp -d) diff --git a/test/lib/common.bash b/test/lib/common.bash index 35c32ef..2189e19 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -127,8 +127,10 @@ setup() { FTP_BASE="${TMP}/ftp" ARCHIVE_BASE="${TMP}/archive" ARCHIVEUSER="" - PKGREPOS=('core' 'extra' 'testing' 'staging') + PKGREPOS=('core' 'extra' 'testing' 'staging' 'noperm') DEBUGREPOS=('core-debug' 'extra-debug' 'testing-debug' 'staging-debug') + ACL=([users]="extra core staging testing core-debug extra-debug testing-debug staging-debug") + PKGPOOL='pool/packages' DEBUGPKGPOOL='pool/packages-debug' SRCPOOL='sources/packages' @@ -146,9 +148,13 @@ setup() { GNUPGHOME="/etc/pacman.d/gnupg" GITREPOS="${TMP}/git-packages" GITREPO="${TMP}/repository" - GITUSER="git-packages" -eot + GITPKGREPOS="${TMP}/git-pkg-repos" + GITUSER="" + if [[ -f "${TMP}/config.override" ]]; then + . "${TMP}/config.override" + fi +eot . config @@ -207,6 +213,15 @@ teardown() { rm -rf "${TMP}" } +enablePermission() { + local repo=$1 + grep ACL "${TMP}/config.local" | sed 's/")/ '"$repo"'")/' > "${TMP}/config.override" +} + +disablePermissionOverride() { + rm -f "${TMP}/config.override" +} + releasePackage() { local repo=$1 local pkgbase=$2 From 8e9a8328acd5a66b5a13afa0795cb7c7b316a5cc Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 21 Sep 2022 21:10:21 +0200 Subject: [PATCH 20/44] test: add git dependency to the test container This makes it more convenient to finally run tests locally via a container instead of natively on an isolated dev environment --- test/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Dockerfile b/test/Dockerfile index 7353eb8..32a8068 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,5 +1,5 @@ FROM docker.io/archlinux/archlinux -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils git RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester From 6f26111650999589345dc07d52ee09d65a95a78f Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 21 Sep 2022 20:41:46 +0200 Subject: [PATCH 21/44] git: purely operate on a cached bare mirror instead of checkouts Even with depth=1 we are better of to simply keep caches of the clones. Introduce a new `config` location to store all repositories inside. Whenever the checkout function is called, we now either clone the repository or fetch the latest deltas. Functions that require the actual PKGBUILD are not operating purely in memory and with virtual file handles instead of dealing with actual file checkouts. --- config.local.git | 2 ++ db-functions | 6 ++++-- db-functions-git | 33 +++++++++++++++++++++------------ db-move | 2 +- db-update | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/config.local.git b/config.local.git index abd0dbf..1fd90ce 100644 --- a/config.local.git +++ b/config.local.git @@ -62,3 +62,5 @@ export GNUPGHOME="/etc/pacman.d/gnupg" GITREPOS="https://gitlab.archlinux.org/archlinux/packaging/packages" GITREPO="/repository" GITUSER="git" + +GITPKGREPOS="/git-repos" diff --git a/db-functions b/db-functions index 8855bc1..b7da50f 100644 --- a/db-functions +++ b/db-functions @@ -565,7 +565,7 @@ check_reproducible() { check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" local pkgver="$(conv_tag $(getpkgver ${1}))" - if ! GNUPGHOME=/etc/pacman.d/gnupg git -C "${WORKDIR}/${pkgbase}" verify-tag "${pkgver}" 1>/dev/null ; then + if ! GNUPGHOME=/etc/pacman.d/gnupg git -C "${GITPKGREPOS}/${pkgbase}" verify-tag "${pkgver}" >/dev/null; then return 1 fi } @@ -576,7 +576,9 @@ check_pkgbuild_checksum(){ local pkgver="$(getpkgver ${1})" local pkgfile_checksum="$(_grep_buildinfo "${1}" "pkgbuild_sha256sum")" - local sum="$(sha256sum "${WORKDIR}/${pkgbase}/PKGBUILD")" + local gittag="$(conv_tag $(getpkgver ${1}))" + + local sum="$(sha256sum <(git -C "${GITPKGREPOS}/${pkgbase}" show "${gittag}":PKGBUILD))" if [[ "$pkgfile_checksum" != "${sum%% *}" ]]; then return 1 fi diff --git a/db-functions-git b/db-functions-git index 0b3915f..2778f78 100644 --- a/db-functions-git +++ b/db-functions-git @@ -25,12 +25,21 @@ arch_git() { } -# The SVN version should just check the path is present.... +# Fetch the package sources into a global cache fetch_pkgbuild() { local pkgbase="${1}" - local tag=$(conv_tag "${2}") - [ -d "$WORKDIR/${pkgbase}" ] || git clone --branch "$tag" --depth 1 "${GITREPOS}/${pkgbase}.git" "$WORKDIR/${pkgbase}" - git -C "${WORKDIR}/${pkgbase}" verify-tag "$tag" + local src="${GITREPOS}/${pkgbase}.git" + local target="${GITPKGREPOS}/${pkgbase}" + if [[ ! -d $target ]]; then + if ! arch_git -c core.sharedRepository=group clone --origin origin --bare --mirror "${src}" "${target}"; then + return 1 + fi + return 0 + fi + if ! arch_git -C "${target}" fetch --prune --prune-tags; then + return 1 + fi + return 0 } @@ -38,7 +47,8 @@ fetch_pkgbuild() { source_pkgbuild() { local pkgbase="$1" local tag=$(conv_tag ${2}) - . <(cat "${WORKDIR}/${pkgbase}/PKGBUILD" 2>/dev/null || echo false) + + . <(arch_git -C "${GITPKGREPOS}/${pkgbase}" show "${tag}":PKGBUILD 2>/dev/null || echo false) } # Export PKGBUILD resource following the same rules as source_pkgbuild() @@ -50,7 +60,7 @@ export_from_pkgrepo() { if [[ ! -e ${dest} ]]; then mkdir -p "${dest}" - arch_git -C "${WORKDIR}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" + arch_git -C "${GITPKGREPOS}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" fi } @@ -82,12 +92,11 @@ vcs_update_package() { local tag="${2}" local gittag="$(conv_tag ${2})" local dest="$3" - if [[ ! -e "$GITREPO/${dest}" ]]; then - die "Repository %s doesn't exist!" "${dest}" - fi - echo "$pkgbase $gittag $(git -C "${WORKDIR}/${pkgbase}" rev-list -n 1 "$gittag")" > "$GITREPO/$dest/$pkgbase" - arch_git -C "${GITREPO}" add "$GITREPO/$dest/$pkgbase" || true - vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" || true + + mkdir -p "${GITREPO}/${dest}" + echo "${pkgbase} ${gittag} $(git -C "${GITPKGREPOS}/${pkgbase}" rev-parse "${gittag}")" > "${GITREPO}/${dest}/${pkgbase}" + arch_git -C "${GITREPO}" add "${GITREPO}/${dest}/${pkgbase}" + vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" } # Write to the VCS in order to track a package moving between different pacman diff --git a/db-move b/db-move index 167fad7..3daad36 100755 --- a/db-move +++ b/db-move @@ -43,7 +43,7 @@ for pkgbase in "${args[@]:2}"; do for pkgarch in "${ARCHES[@]}"; do if vcsrepo_from=$(find_repo_for_package "${pkgbase}" "${pkgarch}" "${repo_from}"); then _pkgver=$(awk '{print $2}' "${GITREPO}/${vcsrepo_from}/${pkgbase}") - if ! fetch_pkgbuild "${pkgbase}" "${_pkgver}"; then + if ! fetch_pkgbuild "${pkgbase}"; then die "Couldn't find package %s in git!" "${pkgbase}" fi read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" ""; \ diff --git a/db-update b/db-update index fe2a1c7..ee0940d 100755 --- a/db-update +++ b/db-update @@ -34,7 +34,7 @@ for repo in "${repos[@]}"; do pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) if (( $? == 0 )); then for pkg in "${pkgs[@]}"; do - if ! fetch_pkgbuild "$(getpkgbase ${pkg})" "$(getpkgver ${pkg})"; then + if ! fetch_pkgbuild "$(getpkgbase "${pkg}")"; then die "Couldn't find package %s in git!" "${pkg}" fi done From fe5a36e2779feb4d28e6d6088c7f50039c196bb4 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 26 Sep 2022 23:27:52 +0200 Subject: [PATCH 22/44] git: properly get pkgver from state repo and die if an entry is missing This fixes several issue with the test suite and also exiting early in case we do not find a matching entry in the current state repository. --- db-functions-git | 7 +++++++ db-move | 11 ++++++++--- db-remove | 12 ++++++------ 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/db-functions-git b/db-functions-git index 2778f78..88c69f3 100644 --- a/db-functions-git +++ b/db-functions-git @@ -81,6 +81,13 @@ find_repo_for_package() { printf '%s\n' "${repos[@]%/}" } +# Get the current pkgver from a given repo +pkgver_from_state_repo() { + local pkgbase=${1} + local repo=${2} + awk '{print $2}' "${GITREPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 +} + # Commit changes staged by (successive?) vcs_(re)?move_package runs. vcs_commit() { arch_git -C "${GITREPO}" commit -m "${1}:" diff --git a/db-move b/db-move index 3daad36..3f1152f 100755 --- a/db-move +++ b/db-move @@ -42,11 +42,13 @@ for pkgbase in "${args[@]:2}"; do found=false for pkgarch in "${ARCHES[@]}"; do if vcsrepo_from=$(find_repo_for_package "${pkgbase}" "${pkgarch}" "${repo_from}"); then - _pkgver=$(awk '{print $2}' "${GITREPO}/${vcsrepo_from}/${pkgbase}") + if ! _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo_from}"); then + die "%s has no pkgver entry in %s" "${pkgbase}" "${vcsrepo_from}" + fi if ! fetch_pkgbuild "${pkgbase}"; then die "Couldn't find package %s in git!" "${pkgbase}" fi - read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" ""; \ + read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "${_pkgver}"; \ get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" if (( ${#pkgnames[@]} < 1 )); then @@ -91,7 +93,10 @@ for pkgbase in "${args[@]:2}"; do tarches=("${pkgarch}") fi msg2 "%s (%s)" "$pkgbase" "${tarches[*]}" - read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "repos/${repo_from}-${pkgarch}"; \ + if ! _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo_from}"); then + die "%s has no entry in %s" "${pkgbase}" "${vcsrepo_from}" + fi + read -rd'\n' pkgver pkgnames < <(source_pkgbuild "${pkgbase}" "${_pkgver}"; \ get_full_version; echo "${pkgname[@]}") read -ra pkgnames <<<"$pkgnames" diff --git a/db-remove b/db-remove index 85180ba..1ecaf33 100755 --- a/db-remove +++ b/db-remove @@ -33,15 +33,15 @@ remove_debug_pkgs=() for pkgbase in "${pkgbases[@]}"; do msg "Removing %s from [%s]..." "$pkgbase" "$repo" - _pkgver=$(cat "${GITREPO}/${vcsrepo}/${pkgbase}"| awk '{print $2}') - fetch_pkgbuild "${pkgbase}" "${_pkgver}" - if remove_pkgs+=($(source_pkgbuild "${pkgbase}" "" && echo ${pkgname[@]})); then + if fetch_pkgbuild "${pkgbase}" && \ + _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo}") && \ + remove_pkgs+=($(source_pkgbuild "${pkgbase}" "${_pkgver}" && printf "%s\n" "${pkgname[@]}")); then vcs_remove_package "${pkgbase}" "${vcsrepo}" vcs_commit "${0##*/}: $pkgbase removed by $(id -un)" else - warning "%s not found in %s" "$pkgbase" "$vcsrepo" - warning "Removing only %s from the repo" "$pkgbase" - warning "If it was a split package you have to remove the others yourself!" + warning "pkgbase %s not found in %s" "$pkgbase" "$vcsrepo" + warning "Removing only pkgname %s from the repo" "$pkgbase" + warning "If it was a split package you have to pass its pkgbase to remove it completely!" remove_pkgs+=("$pkgbase") fi if is_globfile "${FTP_BASE}/${repo}-debug/os/${tarch}/${pkgbase}-debug"*; then From 7fd4275c3c4cede955928ddf6e0165c0517ad665 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 28 Sep 2022 23:31:56 +0200 Subject: [PATCH 23/44] db-update: always pass both, signature and file to gpg verification Only specifying the signature and having gpg rely on assumptions about what to check the signature against is a very bad practice leading to ambiguous possibilities if the signature contains data itself. Make sure that we do not call gpg verification like such by passing both, the signature as well as the actual file of the data to check against. --- db-update | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-update b/db-update index ee0940d..17c496e 100755 --- a/db-update +++ b/db-update @@ -63,7 +63,7 @@ for repo in "${repos[@]}"; do if ! check_pkgfile "${pkg}"; then die "Package %s is not consistent with its meta data" "$repo/${pkg##*/}" fi - if ! pacman-key -v "${pkg}.sig" >/dev/null 2>&1; then + if ! pacman-key --verify "${pkg}.sig" "${pkg}" >/dev/null 2>&1; then die "Package %s does not have a valid signature" "$repo/${pkg##*/}" fi if ! check_pkgvcs "${pkg}" "${repo}"; then From 7279b75b9f892cdf815617e0698aad4715de3e09 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 28 Sep 2022 23:35:59 +0200 Subject: [PATCH 24/44] db-update: fix state repo entries for any packages We do not serve any packages as dedicated architecture repository in pacman, but that doesn't mean we must change the current behavior of dbscripts. A lot of places are build in a way to expect any packages to actually have a dedicated any entry of the matching repository. Lets not try to change this behavior while migrating to git as we already face too much possibility for issues. Hence map any packages to any when changing the state repository. --- db-update | 3 +++ 1 file changed, 3 insertions(+) diff --git a/db-update b/db-update index 17c496e..76de100 100755 --- a/db-update +++ b/db-update @@ -157,6 +157,9 @@ for repo in "${repos[@]}"; do fi if ((REPO_MODIFIED)); then for pkg in "${add_pkgs[@]}"; do + if [[ $pkg == *-any${PKGEXTS} ]]; then + pkgarch=any + fi pkgbase="${add_vcspkgs[$pkg::pkgbase]}" pkgver="${add_vcspkgs[$pkg::pkgver]}" vcs_update_package "$pkgbase" "$pkgver" "${repo}-${pkgarch}" From c9d1cb1b2571eb34d890bfb2ffc63680c11e0f55 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Thu, 29 Sep 2022 22:13:05 +0200 Subject: [PATCH 25/44] db-update: minor code linting, quoting and conditional commands --- db-update | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/db-update b/db-update index 76de100..fbfc605 100755 --- a/db-update +++ b/db-update @@ -31,8 +31,7 @@ done for repo in "${repos[@]}"; do - pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) - if (( $? == 0 )); then + if pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})); then for pkg in "${pkgs[@]}"; do if ! fetch_pkgbuild "$(getpkgbase "${pkg}")"; then die "Couldn't find package %s in git!" "${pkg}" @@ -48,8 +47,7 @@ for repo in "${repos[@]}"; do if ! check_repo_permission "${repo}"; then die "You don't have permission to update packages in %s" "$repo" fi - pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})) - if (( $? == 0 )); then + if pkgs=($(getpkgfiles "${STAGING}/${repo}/"*${PKGEXTS})); then for pkg in "${pkgs[@]}"; do if [[ -h ${pkg} ]]; then die "Package %s is a symbolic link" "$repo/${pkg##*/}" @@ -136,12 +134,13 @@ for repo in "${repos[@]}"; do if [[ -f ${pkg} ]]; then mv "${pkg}" "$FTP_BASE/${currentpool}" fi + mkdir -p "$FTP_BASE/${currentrepo}/os/${pkgarch}" ln -sf "../../../${currentpool}/${pkgfile}" "$FTP_BASE/${currentrepo}/os/${pkgarch}" # also move signatures if [[ -f ${pkg}.sig ]]; then mv "${pkg}.sig" "$FTP_BASE/${currentpool}" fi - if [[ ${PKGPOOL} = ${currentpool} ]]; then + if [[ ${PKGPOOL} = "${currentpool}" ]]; then # do not archive debug info, this is not of historic interest "$(dirname "$(readlink -e "${BASH_SOURCE[0]}")")/db-archive" "${FTP_BASE}/${PKGPOOL}/${pkg##*/}" fi @@ -162,7 +161,7 @@ for repo in "${repos[@]}"; do fi pkgbase="${add_vcspkgs[$pkg::pkgbase]}" pkgver="${add_vcspkgs[$pkg::pkgver]}" - vcs_update_package "$pkgbase" "$pkgver" "${repo}-${pkgarch}" + vcs_update_package "${pkgbase}" "${pkgver}" "${repo}-${pkgarch}" done fi done From ed267105d8de79ac07ee78cf0cafe656832f5b31 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Thu, 29 Sep 2022 22:14:49 +0200 Subject: [PATCH 26/44] config: curated PATH to sanitize executables in a portable way We rely on some safety guards now, as we are doing ACL in code instead of purely relying on filesystem access. Hence lets try to curate the PATH which in return will protect against easily smuggling in maliciously behaving executables we may call during execution. --- config | 4 ++++ test/Dockerfile | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/config b/config index b4144d6..be340a8 100644 --- a/config +++ b/config @@ -49,6 +49,10 @@ LIST="arch-dev-public@lists.archlinux.org" #LIST="aaronmgriffin@gmail.com" FROM="repomaint@archlinux.org" +# curated PATH to sanitize executables in a portable way +PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export PATH + # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname "${BASH_SOURCE[0]}")/config.local"} [[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/test/Dockerfile b/test/Dockerfile index 32a8068..d92d98c 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -3,6 +3,18 @@ RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash RUN pacman-key --init RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel RUN useradd -N -g users -G wheel -d /build -m tester +RUN ln -sf /dbscripts/db-archive /usr/local/bin/ && \ + ln -sf /dbscripts/db-move /usr/local/bin/ && \ + ln -sf /dbscripts/db-remove /usr/local/bin/ && \ + ln -sf /dbscripts/db-repo-add /usr/local/bin/ && \ + ln -sf /dbscripts/db-repo-remove /usr/local/bin/ && \ + ln -sf /dbscripts/db-update /usr/local/bin/ && \ + ln -sf /dbscripts/testing2x /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/devlist-mailer /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/ftpdir-cleanup /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/integrity-check /usr/local/bin/ && \ + ln -sf /dbscripts/cron-jobs/sourceballs /usr/local/bin/ + USER tester RUN echo -e "\ Key-Type: RSA\n\ From e3ee9bb18b50c4d309d6ff6d001c1cdf65e85957 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 1 Oct 2022 02:16:55 +0200 Subject: [PATCH 27/44] db-move: fix state repo entries for any packages We do not serve any packages as dedicated architecture repository in pacman, but that doesn't mean we must change the current behavior of dbscripts. A lot of places are build in a way to expect any packages to actually have a dedicated any entry of the matching repository. Lets not try to change this behavior while migrating to git as we already face too much possibility for issues. Hence map any packages to any when changing the state repository. --- db-move | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db-move b/db-move index 3f1152f..a55a23a 100755 --- a/db-move +++ b/db-move @@ -75,6 +75,7 @@ done msg "Moving packages from [%s] to [%s]..." "$repo_from" "$repo_to" +declare -A pkgbase_arches=() for arch in "${ARCHES[@]}"; do declare -a add_pkgs_$arch declare -a add_debug_pkgs_$arch @@ -101,6 +102,7 @@ for pkgbase in "${args[@]:2}"; do read -ra pkgnames <<<"$pkgnames" tag_list+=", $pkgarch" + pkgbase_arches[$pkgbase]+="$pkgarch " for tarch in "${tarches[@]}"; do declare -n add_pkgs="add_pkgs_${tarch}" @@ -155,7 +157,8 @@ done # We really don't want it to represent a broken state if ((REPO_MODIFIED)); then for pkgbase in "${args[@]:2}"; do - for tarch in "${ARCHES[@]}"; do + read -r -a tarches <<< "${pkgbase_arches[${pkgbase}]}" + for tarch in "${tarches[@]}"; do if ! vcs_move_package "${pkgbase}" "${repo_from}-${tarch}" "${repo_to}-${tarch}"; then die "Couldn't move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" fi From 6a453b9a89723ddd5b4eaae74d352ec67388b769 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 21 Oct 2022 21:35:55 +0200 Subject: [PATCH 28/44] git: simplify state handling by not requiring paths to pre exist It's a lot more convenient for operation if we do not require pre existing state assumptions but rather ensure we initialize whatever path we require during execution of the vcs state mutation functions. Any information we want to retrieve from the history will also not require any paths to exist indefinitely, but instead we follow paths in the history no matter when they appear or disappear in between. --- db-functions-git | 9 +++++---- test/lib/common.bash | 14 ++------------ 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/db-functions-git b/db-functions-git index 88c69f3..274d3e4 100644 --- a/db-functions-git +++ b/db-functions-git @@ -110,10 +110,11 @@ vcs_update_package() { # repositories. vcs_move_package() { local pkgbase=${1} - local vcsrepo_from=${GITREPO}/${2} - local vcsrepo_to=${GITREPO}/${3} - mkdir -p "${vcsrepo_to}" - arch_git -C "$GITREPO" mv "$vcsrepo_from/${pkgbase}" "$vcsrepo_to/${pkgbase}" + local vcsrepo_from=${2} + local vcsrepo_to=${3} + + mkdir -p "${GITREPO}/${vcsrepo_to}" + arch_git -C "${GITREPO}" mv "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" } # Write to the VCS in order to track a package being deleted from a pacman diff --git a/test/lib/common.bash b/test/lib/common.bash index 2189e19..3863cee 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -193,20 +193,10 @@ eot touch "${ARCHIVE_BASE}/packages/${pkgname:0:1}/${pkgname}/${line}"{,.sig} done - git init --bare --shared=group "${TMPDIR}/git-packages-bare.git" + git init --bare --shared=group "${TMPDIR}/git-packages-bare.git" mkdir "${GITREPO}" chmod 777 "${GITREPO}" - arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GITREPO}" - for r in ${PKGREPOS[@]}; do - for a in ${ARCHES[@]}; do - # This is ugly but we need 770 in the test env - (umask 002; - mkdir -p "${GITREPO}/${r}-${a}"; - touch "${GITREPO}/${r}-${a}"/.gitkeep; - arch_git -C "${GITREPO}" add "${GITREPO}/${r}-${a}"/.gitkeep) - done - done - arch_git -C "${GITREPO}" commit -m "init repos" + arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GITREPO}" 2>/dev/null } teardown() { From c0df1e6c0fe9f5f344fc09ec5f094de30606e8d7 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 22 Oct 2022 11:40:07 +0200 Subject: [PATCH 29/44] db-move: fix repo mutation when testing packages overwrite old stables We can safely call with `--force` to force renaming or moving of a file even if the target exists. This is a common case whenever we want to move packages between repositories which will result in overwriting an old stable version from testing. This commit makes a bunch of new tests cases pass and also makes logically a lot of sense to do. --- db-functions-git | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db-functions-git b/db-functions-git index 274d3e4..eccf9b9 100644 --- a/db-functions-git +++ b/db-functions-git @@ -114,7 +114,7 @@ vcs_move_package() { local vcsrepo_to=${3} mkdir -p "${GITREPO}/${vcsrepo_to}" - arch_git -C "${GITREPO}" mv "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" + arch_git -C "${GITREPO}" mv --force "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" } # Write to the VCS in order to track a package being deleted from a pacman From 76c98a5d27ccca8c0a9f0070713f2e24245d467c Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 25 Oct 2022 23:57:17 +0200 Subject: [PATCH 30/44] db-remove: consistency with other vcs functions by implicitly committing We already commit implicitly during move and update function state mutations. Do the same for db-remove operations and provide a uniform behavior across this family of functions. --- db-functions-git | 9 +++------ db-remove | 1 - 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/db-functions-git b/db-functions-git index eccf9b9..d9e5662 100644 --- a/db-functions-git +++ b/db-functions-git @@ -121,11 +121,8 @@ vcs_move_package() { # repository. vcs_remove_package() { local pkgbase=${1} - local vcsrepo=${GITREPO}/${2}/${pkgbase} + local vcsrepo=${2} - arch_git -C "$GITREPO" rm "$vcsrepo" -} - -vcs_push(){ - arch_git -C "$GITREPO" push --tags origin master + arch_git -C "${GITREPO}" rm "${vcsrepo}/${pkgbase}" + vcs_commit "${0##*/}: remove ${pkgbase} from ${vcsrepo}" } diff --git a/db-remove b/db-remove index 1ecaf33..3ecdc72 100755 --- a/db-remove +++ b/db-remove @@ -37,7 +37,6 @@ for pkgbase in "${pkgbases[@]}"; do _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo}") && \ remove_pkgs+=($(source_pkgbuild "${pkgbase}" "${_pkgver}" && printf "%s\n" "${pkgname[@]}")); then vcs_remove_package "${pkgbase}" "${vcsrepo}" - vcs_commit "${0##*/}: $pkgbase removed by $(id -un)" else warning "pkgbase %s not found in %s" "$pkgbase" "$vcsrepo" warning "Removing only pkgname %s from the repo" "$pkgbase" From 688566f4a1ad6f2d9eb0c53f136a888b0e99c1f6 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 29 Oct 2022 00:19:24 +0200 Subject: [PATCH 31/44] db-remove: collect all removed pkgbases before modifying the state repo This commit mostly helps to ensure we act in similar paradigms on the state repository as db-move and db-update does. We may want to revisit this approach though, as this concept can equally lead to inconsistencies between the actual database and the state repo. --- db-remove | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/db-remove b/db-remove index 3ecdc72..f66a1e2 100755 --- a/db-remove +++ b/db-remove @@ -28,6 +28,7 @@ for tarch in "${tarches[@]}"; do repo_lock "$repo" "$tarch" || exit 1 done +remove_pkgbases=() remove_pkgs=() remove_debug_pkgs=() for pkgbase in "${pkgbases[@]}"; do @@ -36,7 +37,7 @@ for pkgbase in "${pkgbases[@]}"; do if fetch_pkgbuild "${pkgbase}" && \ _pkgver=$(pkgver_from_state_repo "${pkgbase}" "${vcsrepo}") && \ remove_pkgs+=($(source_pkgbuild "${pkgbase}" "${_pkgver}" && printf "%s\n" "${pkgname[@]}")); then - vcs_remove_package "${pkgbase}" "${vcsrepo}" + remove_pkgbases+=("${pkgbase}") else warning "pkgbase %s not found in %s" "$pkgbase" "$vcsrepo" warning "Removing only pkgname %s from the repo" "$pkgbase" @@ -56,5 +57,15 @@ for tarch in "${tarches[@]}"; do if (( ${#remove_debug_pkgs[@]} >= 1 )); then arch_repo_modify remove "${repo}-debug" "${tarch}" "${remove_debug_pkgs[@]}" fi +done + +if ((REPO_MODIFIED)); then + for pkgbase in "${remove_pkgbases[@]}"; do + vcs_remove_package "${pkgbase}" "${vcsrepo}" + done +fi + +# Remove all the locks we created +for tarch in "${tarches[@]}"; do repo_unlock "$repo" "$tarch" done From bc685a92a0ac571bf93bb90e22b0907489c637cb Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 15 Nov 2022 23:11:09 +0100 Subject: [PATCH 32/44] git: unset git behavior influencing environment variables This helps to avoid interfering on by accident with expectations of dbscripts by setting environment varialbes that Git uses to change its default setting or behavior. --- db-functions-git | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/db-functions-git b/db-functions-git index d9e5662..ed4e956 100644 --- a/db-functions-git +++ b/db-functions-git @@ -6,6 +6,15 @@ if [[ -n ${GITUSER} ]]; then setfacl -m d:u:"${GITUSER}":rwx "${WORKDIR}" fi +# unset any behavior influencing environment variables +# https://git-scm.com/book/en/v2/Git-Internals-Environment-Variables +while read -r var; do + unset "${var}"; +done < <(env | grep -E '^GIT' | awk -F= '{print $1}') +unset PREFIX +unset EMAIL + + # Converts from the PKGBUILD tag to the git repository tag # Input 1:1.0~0-1 # Output 1-1.0.0-1 From 2689b8d64e870d4c73b958d47fcfbb92cc94684e Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 19 Nov 2022 14:29:15 +0100 Subject: [PATCH 33/44] sourceballs: fix export_from_vcs calls to create package source archives Pass along the correct expected pkgver to export an archive from. This fixes the dbscripts test suite for sourceballs tests. --- cron-jobs/sourceballs | 2 +- db-functions-git | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cron-jobs/sourceballs b/cron-jobs/sourceballs index 5980703..f9ff780 100755 --- a/cron-jobs/sourceballs +++ b/cron-jobs/sourceballs @@ -79,7 +79,7 @@ for repo in "${PKGREPOS[@]}"; do # Get the sources from svn mkdir -p -m0770 "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}" - export_from_vcs "${pkgbase}" "repos/${repo}-${pkgarch}" "" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" + export_from_vcs "${pkgbase}" "${pkgver}" "${WORKDIR}/pkgbuilds/${repo}-${pkgarch}/${pkgbase}" if (( $? >= 1 )); then failedpkgs+=("${pkgbase}-${pkgver}${SRCEXT}") continue diff --git a/db-functions-git b/db-functions-git index ed4e956..0b67e06 100644 --- a/db-functions-git +++ b/db-functions-git @@ -61,11 +61,10 @@ source_pkgbuild() { } # Export PKGBUILD resource following the same rules as source_pkgbuild() -export_from_pkgrepo() { +export_from_vcs() { local pkgbase="$1" local tag=$(conv_tag ${2}) - local src="$3" - local dest="$4" + local dest="$3" if [[ ! -e ${dest} ]]; then mkdir -p "${dest}" From 09b1142dd8c3b8a02e7e4fecfea4f63e840e7ec4 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 19 Nov 2022 15:13:27 +0100 Subject: [PATCH 34/44] test: collapse container layers that do not need to be individual It is discouraged to have every command in an own container layer if there is no benefit in having them separate. For example it makes sense to have a static block before having a dynamic block like system updates hence the first layer will mostly never be rebuild. But that all isn't much required here, lets keep it simple by collapsing the polluted amount of layers into the minimum set. --- test/Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/Dockerfile b/test/Dockerfile index d92d98c..8bdca9e 100644 --- a/test/Dockerfile +++ b/test/Dockerfile @@ -1,9 +1,9 @@ FROM docker.io/archlinux/archlinux -RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils git -RUN pacman-key --init -RUN echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel -RUN useradd -N -g users -G wheel -d /build -m tester -RUN ln -sf /dbscripts/db-archive /usr/local/bin/ && \ +RUN pacman -Syu --noconfirm --needed sudo fakeroot awk subversion make kcov bash-bats gettext grep tree binutils git && \ + pacman-key --init && \ + echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/wheel && \ + useradd -N -g users -G wheel -d /build -m tester && \ + ln -sf /dbscripts/db-archive /usr/local/bin/ && \ ln -sf /dbscripts/db-move /usr/local/bin/ && \ ln -sf /dbscripts/db-remove /usr/local/bin/ && \ ln -sf /dbscripts/db-repo-add /usr/local/bin/ && \ @@ -25,7 +25,7 @@ Name-Email: tester@localhost\n\ Expire-Date: 0\n\ %no-protection\n\ %commit\n"\ -| gpg --quiet --batch --no-tty --no-permission-warning --gen-key -RUN gpg --export | sudo pacman-key -a - -RUN sudo pacman-key --lsign-key tester@localhost +| gpg --quiet --batch --no-tty --no-permission-warning --gen-key && \ + gpg --export | sudo pacman-key -a - && \ + sudo pacman-key --lsign-key tester@localhost ENV PACKAGER="Bob Tester " From 2d58642630791ef9af11aa5668b5bfc71a5d4a5c Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sun, 20 Nov 2022 20:07:55 +0100 Subject: [PATCH 35/44] git: use curated author information resolved by unix username This should be a convenient way to automatically retrieve the author informations for the state repository changes without the need of each individual user to setup yet another author file. This is a central service, hence we can easily deploy and generate the authors file alongside the actual unix user and group permissions rolled out by ansible. --- config | 2 ++ db-functions | 38 ++++++++++++++++++++++++++++++++++++++ db-functions-git | 18 +++++++++++++++++- db-move | 4 ++++ db-remove | 3 +++ db-update | 3 +++ test/cases/db-move.bats | 23 +++++++++++++++++++++++ test/cases/db-remove.bats | 21 +++++++++++++++++++++ test/cases/db-update.bats | 19 +++++++++++++++++++ test/lib/common.bash | 28 ++++++++++++++++++++++++++++ 10 files changed, 158 insertions(+), 1 deletion(-) diff --git a/config b/config index be340a8..4354d2b 100644 --- a/config +++ b/config @@ -53,6 +53,8 @@ FROM="repomaint@archlinux.org" PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH +AUTHORS=/notexists + # Override default config with config.local LOCAL_CONFIG=${DBSCRIPTS_CONFIG:-"$(dirname "${BASH_SOURCE[0]}")/config.local"} [[ -f "${LOCAL_CONFIG}" ]] && . "${LOCAL_CONFIG}" diff --git a/db-functions b/db-functions index b7da50f..923c28b 100644 --- a/db-functions +++ b/db-functions @@ -295,6 +295,44 @@ check_builddir() { [[ -n $_builddir && $_builddir = '/build' ]] } +check_author() { + local author + if ! [[ -f "${AUTHORS}" ]]; then + die "No such authors file: '%s'" "${AUTHORS}" + return 1 + fi + if ! author=$(get_author); then + return 1 + fi + return 0 +} + +get_author() { + if ! username=$(/usr/bin/id -un); then + die "Failed to get username from 'id'" + fi + if ! author=$(grep -E " ${username}\$" "${AUTHORS}"); then + die "Missing author information for %s in '%s'" "${username}" "${AUTHORS}" + fi + echo "${author}" +} + +get_author_name() { + local author=$1 + if ! name=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\1/'); then + die "Failed to get name from author line" + fi + echo "${name}" +} + +get_author_email() { + local author=$1 + if ! email=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\2/'); then + die "Failed to get name from author line" + fi + echo "${email}" +} + # Non fatal getpkgfile expanding globs maybe_getpkgfile() { if (( $# != 1 )); then diff --git a/db-functions-git b/db-functions-git index 0b67e06..89b1bcc 100644 --- a/db-functions-git +++ b/db-functions-git @@ -98,7 +98,23 @@ pkgver_from_state_repo() { # Commit changes staged by (successive?) vcs_(re)?move_package runs. vcs_commit() { - arch_git -C "${GITREPO}" commit -m "${1}:" + local commit_message=$1 + local username author name email + + if ! username=$(/usr/bin/id -un); then + die "Failed to get username from 'id'" + fi + if ! author=$(grep -E " ${username}\$" "${AUTHORS}"); then + die "Missing author information for %s in '%s'" "${username}" "${AUTHORS}" + fi + if ! name=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\1/'); then + die "Failed to extract name from author line" + fi + if ! email=$(echo "${author}"|sed -E 's/(.+) (<.+>) .+/\2/'); then + die "Failed to extract email from author line" + fi + + arch_git -c user.name="${name}" -c user.email="${email}" -C "${GITREPO}" commit -m "${commit_message}" } diff --git a/db-move b/db-move index a55a23a..1fa1db3 100755 --- a/db-move +++ b/db-move @@ -31,6 +31,10 @@ if ! check_repo_permission "$repo_to" || ! check_repo_permission "$repo_from"; t die "You don't have permission to move packages from %s to %s" "$repo_from" "$repo_to" fi +if ! check_author; then + die "You don't have a matching author mapping" +fi + # TODO: this might lock too much (architectures) for pkgarch in "${ARCHES[@]}"; do repo_lock "${repo_to}" "${pkgarch}" || exit 1 diff --git a/db-remove b/db-remove index f66a1e2..007efba 100755 --- a/db-remove +++ b/db-remove @@ -17,6 +17,9 @@ vcsrepo="$repo-$arch" if ! check_repo_permission "$repo"; then die "You don't have permission to remove packages from %s" "$repo" fi +if ! check_author; then + die "You don't have a matching author mapping" +fi if [[ $arch = any ]]; then tarches=("${ARCHES[@]}") diff --git a/db-update b/db-update index fbfc605..912b913 100755 --- a/db-update +++ b/db-update @@ -41,6 +41,9 @@ for repo in "${repos[@]}"; do done +if ! check_author; then + die "You don't have a matching author mapping" +fi # check if packages are valid for repo in "${repos[@]}"; do diff --git a/test/cases/db-move.bats b/test/cases/db-move.bats index 64d8015..e214198 100644 --- a/test/cases/db-move.bats +++ b/test/cases/db-move.bats @@ -237,3 +237,26 @@ load ../lib/common checkPackage noperm ${pkgbase} 1-1 done } + +@test "move package with author mapping" { + releasePackage testing pkg-any-a + db-update + + db-move testing extra pkg-any-a + + checkPackage extra pkg-any-a 1-1 + checkRemovedPackage testing pkg-any-a + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "move package with missing author mapping fails" { + releasePackage testing pkg-any-a + db-update + + emptyAuthorsFile + run db-move testing extra pkg-any-a + [ "$status" -ne 0 ] + + checkPackage testing pkg-any-a 1-1 + checkRemovedPackage extra pkg-any-a +} diff --git a/test/cases/db-remove.bats b/test/cases/db-remove.bats index 9aadffb..18e0e26 100644 --- a/test/cases/db-remove.bats +++ b/test/cases/db-remove.bats @@ -147,3 +147,24 @@ load ../lib/common checkPackage noperm ${pkgbase} 1-1 } + +@test "remove package with author mapping" { + releasePackage testing pkg-any-a + db-update + + db-remove testing any pkg-any-a + + checkRemovedPackage testing pkg-any-a + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "remove package with missing author mapping fails" { + releasePackage testing pkg-any-a + db-update + + emptyAuthorsFile + run db-remove testing any pkg-any-a + [ "$status" -ne 0 ] + + checkPackage testing pkg-any-a 1-1 +} diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 00f86b9..1f02ed1 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -321,3 +321,22 @@ load ../lib/common checkPackage extra-debug ${pkgbase} 1-1 done } + +@test "add package with author mapping" { + releasePackage extra pkg-any-a + + db-update + + checkPackage extra pkg-any-a 1-1 + checkStateRepoAutoredBy "Cake Foobar " +} + +@test "add package with missing author mapping fails" { + releasePackage extra pkg-any-a + + emptyAuthorsFile + run db-update + [ "$status" -ne 0 ] + + checkRemovedPackage extra pkg-any-a +} diff --git a/test/lib/common.bash b/test/lib/common.bash index 3863cee..9e2c691 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -117,6 +117,7 @@ setup() { local pkg local r local a + local username PKGEXT=".pkg.tar.xz" TMP="$(mktemp -d)" @@ -150,12 +151,21 @@ setup() { GITREPO="${TMP}/repository" GITPKGREPOS="${TMP}/git-pkg-repos" GITUSER="" + AUTHORS="${TMP}/authors.conf" if [[ -f "${TMP}/config.override" ]]; then . "${TMP}/config.override" fi eot + username=$(/usr/bin/id -un) + cat < "${TMP}/authors.conf" + qux dux + Cake Foobar ${username} + muh cow + ${username} <${username}@yay> doo +eot + . config git config --global user.email "tester@localhost" @@ -238,6 +248,10 @@ releasePackage() { popd } +emptyAuthorsFile() { + echo > "${TMP}/authors.conf" +} + updatePackage() { local pkgbase=$1 @@ -399,3 +413,17 @@ checkRemovedPackageDB() { done done } + +checkStateRepoAutoredBy() { + local expected=$1 + local author + + if ! author=$(git -C "${GITREPO}" show -s --format='%an <%ae>' HEAD); then + die 'Failed to query author of state repository' + fi + if [[ "${expected}" != "${author}" ]]; then + error "Author doesn't match, expected: '%s', actual: '%s'" "${expected}" "${author}" + return 1 + fi + return 0 +} From 1b3723f514c74856ae719200b8e214d8dc89bfc9 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 21 Nov 2022 02:11:38 +0100 Subject: [PATCH 36/44] test: check add package with inconsistent pkgbuild in tag fails We never want to release a package that has inconsistency between the used PKGBUILD and the tagged PKGBUILD. --- test/cases/db-update.bats | 11 ++++++++++- test/lib/common.bash | 28 ++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/test/cases/db-update.bats b/test/cases/db-update.bats index 1f02ed1..4db8962 100644 --- a/test/cases/db-update.bats +++ b/test/cases/db-update.bats @@ -245,11 +245,20 @@ load ../lib/common checkRemovedPackageDB extra 'pkg-any-a' } -@test "add package with inconsistent pkgbuild fails" { +@test "add package with inconsistent pkgbuild in branch succeeds" { releasePackage extra 'pkg-any-a' updateRepoPKGBUILD 'pkg-any-a' extra any + db-update + checkPackage extra 'pkg-any-a' 1-1 +} + +@test "add package with inconsistent pkgbuild in tag fails" { + releasePackage extra 'pkg-any-a' + + retagModifiedPKGBUILD 'pkg-any-a' + run db-update [ "$status" -ne 0 ] checkRemovedPackageDB extra 'pkg-any-a' diff --git a/test/lib/common.bash b/test/lib/common.bash index 9e2c691..24d9bf9 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -21,6 +21,26 @@ __updatePKGBUILD() { git push } +__retagModifiedPKGBUILD() { + local pkgver + local pkgrel + local gittag + + pkgver=$(. PKGBUILD; echo "${pkgrel}") + pkgrel=$(. PKGBUILD; echo "${pkgrel}") + gittag="${pkgver}-${pkgrel}" + + echo >> PKGBUILD + git add PKGBUILD + git commit -m "modified PKGBUILD" + + # re-tag + git push origin :"${gittag}" + git tag -d "${gittag}" + git tag -s -m "released ${gittag}" "${gittag}" + git push --tags origin main +} + __getCheckSum() { local result result="$(sha1sum "$1")" @@ -272,6 +292,14 @@ updateRepoPKGBUILD() { popd } +retagModifiedPKGBUILD() { + local pkgbase=$1 + + pushd "${TMP_WORKDIR_GIT}/${pkgbase}" + __retagModifiedPKGBUILD + popd +} + checkPackageDB() { local repo=$1 local pkgbase=$2 From d41a38293b0eea19f6f0a842f323ae720e9eda7d Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Tue, 22 Nov 2022 00:03:41 +0100 Subject: [PATCH 37/44] git: use KEYRING config variable to export GNUPGHOME This way we have better control and isolation of the keyring between dbscripts internal sub command env and test suite gpg handling. Export the GNUPGHOME variable inside the db-functions prolog to ensure any gpg or git operations requiring a keyring operate on the one defined in the config. However doing it this way will not influence any custom calls in the test suite except if explicitly desired. Furthermore remove all explicit passes of GNUPGHOME to subcommands. --- config.local.git | 2 +- db-functions | 5 ++++- test/lib/common.bash | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/config.local.git b/config.local.git index 1fd90ce..c9123e3 100644 --- a/config.local.git +++ b/config.local.git @@ -58,7 +58,7 @@ CLEANUP_DESTDIR="/srv/repos/svn-packages/package-cleanup" SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" TMPDIR="/srv/repos/svn-packages/tmp" -export GNUPGHOME="/etc/pacman.d/gnupg" +KEYRING="/etc/pacman.d/gnupg" GITREPOS="https://gitlab.archlinux.org/archlinux/packaging/packages" GITREPO="/repository" GITUSER="git" diff --git a/db-functions b/db-functions index 923c28b..ceddb60 100644 --- a/db-functions +++ b/db-functions @@ -42,6 +42,9 @@ WORKDIR=$(mktemp -dt "${0##*/}.XXXXXXXXXX") LOCKS=() REPO_MODIFIED=0 +# set static gnupg directory for trusted keyring used by GnuPG, Git etc. +export GNUPGHOME="${KEYRING}" + script_lock() { local LOCKDIR="$TMPDIR/.scriptlock.${0##*/}" if ! mkdir "$LOCKDIR" >/dev/null 2>&1 ; then @@ -603,7 +606,7 @@ check_reproducible() { check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" local pkgver="$(conv_tag $(getpkgver ${1}))" - if ! GNUPGHOME=/etc/pacman.d/gnupg git -C "${GITPKGREPOS}/${pkgbase}" verify-tag "${pkgver}" >/dev/null; then + if ! git -C "${GITPKGREPOS}/${pkgbase}" verify-tag "${pkgver}" >/dev/null 2>&1; then return 1 fi } diff --git a/test/lib/common.bash b/test/lib/common.bash index 24d9bf9..ba9bf22 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -166,7 +166,7 @@ setup() { CLEANUP_DRYRUN=false SOURCE_CLEANUP_DRYRUN=false VCS=git - GNUPGHOME="/etc/pacman.d/gnupg" + KEYRING="/etc/pacman.d/gnupg" GITREPOS="${TMP}/git-packages" GITREPO="${TMP}/repository" GITPKGREPOS="${TMP}/git-pkg-repos" From 6ccf765f05280e1078dd0a687b5ac6d2451c617e Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 23 Nov 2022 19:49:23 +0100 Subject: [PATCH 38/44] db-move: consistency with other vcs functions by implicitly committing We already commit implicitly during remove and update function state mutations. Do the same for db-move operations and provide a uniform behavior across this family of functions. --- db-functions-git | 1 + db-move | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/db-functions-git b/db-functions-git index 89b1bcc..d06e293 100644 --- a/db-functions-git +++ b/db-functions-git @@ -139,6 +139,7 @@ vcs_move_package() { mkdir -p "${GITREPO}/${vcsrepo_to}" arch_git -C "${GITREPO}" mv --force "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" + vcs_commit "move ${pkgbase} from ${vcsrepo_from} to ${vcsrepo_to}" } # Write to the VCS in order to track a package being deleted from a pacman diff --git a/db-move b/db-move index 1fa1db3..9695b8c 100755 --- a/db-move +++ b/db-move @@ -166,9 +166,6 @@ if ((REPO_MODIFIED)); then if ! vcs_move_package "${pkgbase}" "${repo_from}-${tarch}" "${repo_to}-${tarch}"; then die "Couldn't move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" fi - if ! vcs_commit "${0##*/}: moved ${pkgbase} from [${repo_from}] to [${repo_to}] (${tag_list})"; then - die "Couldn't commit the move %s from %s to %s" "${pkgbase}" "${repo_from}" "${repo_to}" - fi done done fi From b86cec2c540dcc3316697a60d2094861018c4977 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 23 Nov 2022 19:54:40 +0100 Subject: [PATCH 39/44] git: more uniform vcs state repo move commit messages --- db-functions-git | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db-functions-git b/db-functions-git index d06e293..31edf1e 100644 --- a/db-functions-git +++ b/db-functions-git @@ -127,7 +127,7 @@ vcs_update_package() { mkdir -p "${GITREPO}/${dest}" echo "${pkgbase} ${gittag} $(git -C "${GITPKGREPOS}/${pkgbase}" rev-parse "${gittag}")" > "${GITREPO}/${dest}/${pkgbase}" arch_git -C "${GITREPO}" add "${GITREPO}/${dest}/${pkgbase}" - vcs_commit "updated ${pkgbase}-${tag} in [${dest}]" + vcs_commit "update ${pkgbase} to ${pkgver} in ${dest}" } # Write to the VCS in order to track a package moving between different pacman @@ -149,5 +149,5 @@ vcs_remove_package() { local vcsrepo=${2} arch_git -C "${GITREPO}" rm "${vcsrepo}/${pkgbase}" - vcs_commit "${0##*/}: remove ${pkgbase} from ${vcsrepo}" + vcs_commit "remove ${pkgbase} from ${vcsrepo}" } From 501be86222747fa8302523101b586576cb80c776 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 23 Nov 2022 20:03:13 +0100 Subject: [PATCH 40/44] git: write original pkgver alongside the state repo metadata The git tag derived from the pkgver is a lossy conversion, lets save the original full pkgver as well so we have a lossless original reference to it. --- db-functions-git | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/db-functions-git b/db-functions-git index 31edf1e..0a8282a 100644 --- a/db-functions-git +++ b/db-functions-git @@ -93,7 +93,22 @@ find_repo_for_package() { pkgver_from_state_repo() { local pkgbase=${1} local repo=${2} - awk '{print $2}' "${GITREPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 + property_from_state_repo "${pkgbase}" "${repo}" 2 +} + +# Get the current git tag name from a given repo +gittag_from_state_repo() { + local pkgbase=${1} + local repo=${2} + property_from_state_repo "${pkgbase}" "${repo}" 3 +} + +# Get the given property position from a given repo +property_from_state_repo() { + local pkgbase=${1} + local repo=${2} + local prop=${3} + awk '{print $'"${prop}"'}' "${GITREPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 } # Commit changes staged by (successive?) vcs_(re)?move_package runs. @@ -120,12 +135,19 @@ vcs_commit() { vcs_update_package() { local pkgbase="$1" - local tag="${2}" - local gittag="$(conv_tag ${2})" + local pkgver="$2" local dest="$3" + local gittag + gittag="$(conv_tag "${pkgver}")" mkdir -p "${GITREPO}/${dest}" - echo "${pkgbase} ${gittag} $(git -C "${GITPKGREPOS}/${pkgbase}" rev-parse "${gittag}")" > "${GITREPO}/${dest}/${pkgbase}" + printf '%s %s %s %s\n' \ + "${pkgbase}" \ + "${pkgver}" \ + "${gittag}" \ + "$(git -C "${GITPKGREPOS}/${pkgbase}" rev-parse "${gittag}")" \ + > "${GITREPO}/${dest}/${pkgbase}" + arch_git -C "${GITREPO}" add "${GITREPO}/${dest}/${pkgbase}" vcs_commit "update ${pkgbase} to ${pkgver} in ${dest}" } From 4d64fca90bf5d7e1f4ff660e20288aa7d1899af7 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Thu, 29 Sep 2022 22:30:08 +0200 Subject: [PATCH 41/44] WIP: disable reproducible checks -- TESTING ONLY, DROP ON RELEASE --- db-update | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db-update b/db-update index 912b913..b9b0537 100755 --- a/db-update +++ b/db-update @@ -85,10 +85,10 @@ for repo in "${repos[@]}"; do if ! check_builddir "${pkg}"; then die "Package %s was not built in a chroot" "$repo/${pkg##*/}" fi - if ! check_reproducible "${pkg}"; then - error "Package %s is not reproducible." "${pkg}" - die "Ensure that all dependencies are available in the repositories or are added in the same db-update." - fi + #if ! check_reproducible "${pkg}"; then + # error "Package %s is not reproducible." "${pkg}" + # die "Ensure that all dependencies are available in the repositories or are added in the same db-update." + #fi done if ! check_splitpkgs "${repo}" "${pkgs[@]}"; then die "Missing split packages for %s" "$repo" From 5065873271efb83912810e065922386bc7d12657 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 23 Nov 2022 20:53:29 +0100 Subject: [PATCH 42/44] cleanup: rename conv_tag to gittag_from_pkgver for readability --- db-functions | 4 ++-- db-functions-git | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/db-functions b/db-functions index ceddb60..4c4b2f6 100644 --- a/db-functions +++ b/db-functions @@ -605,7 +605,7 @@ check_reproducible() { # TODO: Needs to verify the keyid is in the keyring check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" - local pkgver="$(conv_tag $(getpkgver ${1}))" + local pkgver="$(gittag_from_pkgver "$(getpkgver "${1}")")" if ! git -C "${GITPKGREPOS}/${pkgbase}" verify-tag "${pkgver}" >/dev/null 2>&1; then return 1 fi @@ -617,7 +617,7 @@ check_pkgbuild_checksum(){ local pkgver="$(getpkgver ${1})" local pkgfile_checksum="$(_grep_buildinfo "${1}" "pkgbuild_sha256sum")" - local gittag="$(conv_tag $(getpkgver ${1}))" + local gittag="$(gittag_from_pkgver "$(getpkgver "${1}")")" local sum="$(sha256sum <(git -C "${GITPKGREPOS}/${pkgbase}" show "${gittag}":PKGBUILD))" if [[ "$pkgfile_checksum" != "${sum%% *}" ]]; then diff --git a/db-functions-git b/db-functions-git index 0a8282a..27d9522 100644 --- a/db-functions-git +++ b/db-functions-git @@ -18,11 +18,12 @@ unset EMAIL # Converts from the PKGBUILD tag to the git repository tag # Input 1:1.0~0-1 # Output 1-1.0.0-1 -conv_tag() { - local git_tag="$1" - git_tag="${git_tag/:/-}" - git_tag="${git_tag//\~/.}" - printf "%s" "${git_tag}" +gittag_from_pkgver() { + local pkgver="$1" + local gittag + gittag="${pkgver/:/-}" + gittag="${gittag//\~/.}" + printf "%s" "${gittag}" } arch_git() { @@ -55,7 +56,7 @@ fetch_pkgbuild() { # Source the PKGBUILD from the package's git/svn/whatever repo. source_pkgbuild() { local pkgbase="$1" - local tag=$(conv_tag ${2}) + local tag=$(gittag_from_pkgver "${2}") . <(arch_git -C "${GITPKGREPOS}/${pkgbase}" show "${tag}":PKGBUILD 2>/dev/null || echo false) } @@ -63,7 +64,7 @@ source_pkgbuild() { # Export PKGBUILD resource following the same rules as source_pkgbuild() export_from_vcs() { local pkgbase="$1" - local tag=$(conv_tag ${2}) + local tag=$(gittag_from_pkgver "${2}") local dest="$3" if [[ ! -e ${dest} ]]; then @@ -138,7 +139,7 @@ vcs_update_package() { local pkgver="$2" local dest="$3" local gittag - gittag="$(conv_tag "${pkgver}")" + gittag="$(gittag_from_pkgver "${pkgver}")" mkdir -p "${GITREPO}/${dest}" printf '%s %s %s %s\n' \ From ace58717ad48fee3f2f07da2cd9b00c55c903b54 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Wed, 23 Nov 2022 21:42:50 +0100 Subject: [PATCH 43/44] cleanup: rename git cache, state and url variables for readability --- config.local.git | 8 ++++---- db-functions | 4 ++-- db-functions-git | 28 ++++++++++++++-------------- test/lib/common.bash | 30 +++++++++++++++--------------- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/config.local.git b/config.local.git index c9123e3..2abb190 100644 --- a/config.local.git +++ b/config.local.git @@ -59,8 +59,8 @@ SOURCE_CLEANUP_DESTDIR="/srv/repos/svn-packages/source-cleanup" TMPDIR="/srv/repos/svn-packages/tmp" KEYRING="/etc/pacman.d/gnupg" -GITREPOS="https://gitlab.archlinux.org/archlinux/packaging/packages" -GITREPO="/repository" -GITUSER="git" +GIT_PACKAGING_REPOS_URL="https://gitlab.archlinux.org/archlinux/packaging/packages" +GIT_STATE_REPO="/srv/repos/state" +GITUSER="" -GITPKGREPOS="/git-repos" +GIT_PACKAGES_CACHE="/srv/repos/pkg-cache" diff --git a/db-functions b/db-functions index 4c4b2f6..203891c 100644 --- a/db-functions +++ b/db-functions @@ -606,7 +606,7 @@ check_reproducible() { check_signed_tag(){ local pkgbase="$(getpkgbase ${1})" local pkgver="$(gittag_from_pkgver "$(getpkgver "${1}")")" - if ! git -C "${GITPKGREPOS}/${pkgbase}" verify-tag "${pkgver}" >/dev/null 2>&1; then + if ! git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" verify-tag "${pkgver}" >/dev/null 2>&1; then return 1 fi } @@ -619,7 +619,7 @@ check_pkgbuild_checksum(){ local gittag="$(gittag_from_pkgver "$(getpkgver "${1}")")" - local sum="$(sha256sum <(git -C "${GITPKGREPOS}/${pkgbase}" show "${gittag}":PKGBUILD))" + local sum="$(sha256sum <(git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" show "${gittag}":PKGBUILD))" if [[ "$pkgfile_checksum" != "${sum%% *}" ]]; then return 1 fi diff --git a/db-functions-git b/db-functions-git index 27d9522..8c9bd5c 100644 --- a/db-functions-git +++ b/db-functions-git @@ -38,8 +38,8 @@ arch_git() { # Fetch the package sources into a global cache fetch_pkgbuild() { local pkgbase="${1}" - local src="${GITREPOS}/${pkgbase}.git" - local target="${GITPKGREPOS}/${pkgbase}" + local src="${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" + local target="${GIT_PACKAGES_CACHE}/${pkgbase}" if [[ ! -d $target ]]; then if ! arch_git -c core.sharedRepository=group clone --origin origin --bare --mirror "${src}" "${target}"; then return 1 @@ -58,7 +58,7 @@ source_pkgbuild() { local pkgbase="$1" local tag=$(gittag_from_pkgver "${2}") - . <(arch_git -C "${GITPKGREPOS}/${pkgbase}" show "${tag}":PKGBUILD 2>/dev/null || echo false) + . <(arch_git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" show "${tag}":PKGBUILD 2>/dev/null || echo false) } # Export PKGBUILD resource following the same rules as source_pkgbuild() @@ -69,7 +69,7 @@ export_from_vcs() { if [[ ! -e ${dest} ]]; then mkdir -p "${dest}" - arch_git -C "${GITPKGREPOS}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" + arch_git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" archive "$tag" | bsdtar -xf - -C "${dest}" fi } @@ -79,7 +79,7 @@ find_repo_for_package() { local pkgarch=${2} local candidates=("${@:3}") - local repos=($(arch_git -C "${GITREPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ + local repos=($(arch_git -C "${GIT_STATE_REPO}" ls-files "*/$pkgbase" | awk -F/ '{print $1}' | \ grep -xFf <(printf "%s\n" "${candidates[@]/%/-${pkgarch}}" "${candidates[@]/%/-any}"))) if (( ${#repos[@]} > 1 )); then @@ -109,7 +109,7 @@ property_from_state_repo() { local pkgbase=${1} local repo=${2} local prop=${3} - awk '{print $'"${prop}"'}' "${GITREPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 + awk '{print $'"${prop}"'}' "${GIT_STATE_REPO}/${repo}/${pkgbase}" 2>/dev/null || return 1 } # Commit changes staged by (successive?) vcs_(re)?move_package runs. @@ -130,7 +130,7 @@ vcs_commit() { die "Failed to extract email from author line" fi - arch_git -c user.name="${name}" -c user.email="${email}" -C "${GITREPO}" commit -m "${commit_message}" + arch_git -c user.name="${name}" -c user.email="${email}" -C "${GIT_STATE_REPO}" commit -m "${commit_message}" } @@ -141,15 +141,15 @@ vcs_update_package() { local gittag gittag="$(gittag_from_pkgver "${pkgver}")" - mkdir -p "${GITREPO}/${dest}" + mkdir -p "${GIT_STATE_REPO}/${dest}" printf '%s %s %s %s\n' \ "${pkgbase}" \ "${pkgver}" \ "${gittag}" \ - "$(git -C "${GITPKGREPOS}/${pkgbase}" rev-parse "${gittag}")" \ - > "${GITREPO}/${dest}/${pkgbase}" + "$(git -C "${GIT_PACKAGES_CACHE}/${pkgbase}" rev-parse "${gittag}")" \ + > "${GIT_STATE_REPO}/${dest}/${pkgbase}" - arch_git -C "${GITREPO}" add "${GITREPO}/${dest}/${pkgbase}" + arch_git -C "${GIT_STATE_REPO}" add "${GIT_STATE_REPO}/${dest}/${pkgbase}" vcs_commit "update ${pkgbase} to ${pkgver} in ${dest}" } @@ -160,8 +160,8 @@ vcs_move_package() { local vcsrepo_from=${2} local vcsrepo_to=${3} - mkdir -p "${GITREPO}/${vcsrepo_to}" - arch_git -C "${GITREPO}" mv --force "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" + mkdir -p "${GIT_STATE_REPO}/${vcsrepo_to}" + arch_git -C "${GIT_STATE_REPO}" mv --force "${vcsrepo_from}/${pkgbase}" "${vcsrepo_to}/${pkgbase}" vcs_commit "move ${pkgbase} from ${vcsrepo_from} to ${vcsrepo_to}" } @@ -171,6 +171,6 @@ vcs_remove_package() { local pkgbase=${1} local vcsrepo=${2} - arch_git -C "${GITREPO}" rm "${vcsrepo}/${pkgbase}" + arch_git -C "${GIT_STATE_REPO}" rm "${vcsrepo}/${pkgbase}" vcs_commit "remove ${pkgbase} from ${vcsrepo}" } diff --git a/test/lib/common.bash b/test/lib/common.bash index ba9bf22..953b2e1 100644 --- a/test/lib/common.bash +++ b/test/lib/common.bash @@ -167,9 +167,9 @@ setup() { SOURCE_CLEANUP_DRYRUN=false VCS=git KEYRING="/etc/pacman.d/gnupg" - GITREPOS="${TMP}/git-packages" - GITREPO="${TMP}/repository" - GITPKGREPOS="${TMP}/git-pkg-repos" + GIT_PACKAGING_REPOS_URL="${TMP}/git-packages" + GIT_STATE_REPO="${TMP}/repository" + GIT_PACKAGES_CACHE="${TMP}/git-pkg-repos" GITUSER="" AUTHORS="${TMP}/authors.conf" @@ -198,7 +198,7 @@ eot TMP_WORKDIR_GIT=${TMP}/git-clones mkdir -p "${TMP}/"{ftp,tmp,staging,{package,source}-cleanup} - mkdir -p "${GITREPOS}" + mkdir -p "${GIT_PACKAGING_REPOS_URL}" mkdir -p "${TMP_WORKDIR_GIT}" for r in ${PKGREPOS[@]}; do @@ -224,9 +224,9 @@ eot done git init --bare --shared=group "${TMPDIR}/git-packages-bare.git" - mkdir "${GITREPO}" - chmod 777 "${GITREPO}" - arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GITREPO}" 2>/dev/null + mkdir "${GIT_STATE_REPO}" + chmod 777 "${GIT_STATE_REPO}" + arch_git -c "core.sharedRepository=group" clone "${TMPDIR}/git-packages-bare.git" "${GIT_STATE_REPO}" 2>/dev/null } teardown() { @@ -246,9 +246,9 @@ releasePackage() { local repo=$1 local pkgbase=$2 - if [ ! -d "${GITREPOS}/${pkgbase}.git" ]; then - git init --bare --shared=all "${GITREPOS}/${pkgbase}".git - git -c "core.sharedRepository=group" clone "${GITREPOS}/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" + if [ ! -d "${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" ]; then + git init --bare --shared=all "${GIT_PACKAGING_REPOS_URL}/${pkgbase}".git + git -c "core.sharedRepository=group" clone "${GIT_PACKAGING_REPOS_URL}/${pkgbase}".git "${TMP_WORKDIR_GIT}/${pkgbase}" cp -r "fixtures/${pkgbase}"/* "${TMP_WORKDIR_GIT}/${pkgbase}" git -C "${TMP_WORKDIR_GIT}/${pkgbase}" add "${TMP_WORKDIR_GIT}/${pkgbase}"/* git -C "${TMP_WORKDIR_GIT}/${pkgbase}" commit -m "initial commit of ${pkgbase}" @@ -257,14 +257,14 @@ releasePackage() { fi if [ ! -d "${TMP_WORKDIR_GIT}/${pkgbase}" ]; then - git clone "${GITREPOS}/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" + git clone --origin origin "${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" "${TMP_WORKDIR_GIT}/${pkgbase}" fi pushd "${TMP_WORKDIR_GIT}/${pkgbase}" git pull origin main __buildPackage "${STAGING}"/${repo} __archrelease ${repo} - chmod -R 777 "${GITREPOS}/" + chmod -R 777 "${GIT_PACKAGING_REPOS_URL}/" popd } @@ -366,7 +366,7 @@ checkPackage() { local dirarches=() pkgbuildarches=() local pkgbuild dirarch pkgbuildver - for pkgbuild in "${GITREPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; do + for pkgbuild in "${GIT_STATE_REPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; do [[ -e $pkgbuild ]] || continue dirarch=${pkgbuild%/${pkgbase}} dirarch=${dirarch##*-} @@ -395,7 +395,7 @@ checkRemovedPackage() { local repo=$1 local pkgbase=$2 - if __isGlobfile "${GITREPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; then + if __isGlobfile "${GIT_STATE_REPO}/${repo%-debug}-"+([^-])"/${pkgbase}"; then return 1 fi @@ -446,7 +446,7 @@ checkStateRepoAutoredBy() { local expected=$1 local author - if ! author=$(git -C "${GITREPO}" show -s --format='%an <%ae>' HEAD); then + if ! author=$(git -C "${GIT_STATE_REPO}" show -s --format='%an <%ae>' HEAD); then die 'Failed to query author of state repository' fi if [[ "${expected}" != "${author}" ]]; then From eea4cef9d6a66d1f31788d88bacd5594861aebd3 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Mon, 30 Jan 2023 21:19:44 +0100 Subject: [PATCH 44/44] gitlab: add project path function to map special chars Automatic path conversion is limited to GitLab API v4 and will be removed in the future. It's expected that the caller does the path conversion on caller side and only passes a valid path to the API within its limitations. Hence convert project names to valid paths: 1. replace single '+' between word boundaries with '-' 2. replace any other '+' with literal 'plus' 3. replace any special chars other than '_', '-' and '.' with '-' Signed-off-by: Levente Polyak --- db-functions-git | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/db-functions-git b/db-functions-git index 8c9bd5c..cbfa9fe 100644 --- a/db-functions-git +++ b/db-functions-git @@ -34,12 +34,24 @@ arch_git() { fi } +# 1. replace single '+' between word boundaries with '-' +# 2. replace any other '+' with literal 'plus' +# 3. replace any special chars other than '_', '-' and '.' with '-' +gitlab_project_name_to_path() { + local name=$1 + printf "%s" "${name}" \ + | sed -E 's/([a-zA-Z0-9]+)\+([a-zA-Z]+)/\1-\2/g' \ + | sed -E 's/\+/plus/g' \ + | sed -E 's/[^a-zA-Z0-9_\-\.]/-/g' +} # Fetch the package sources into a global cache fetch_pkgbuild() { local pkgbase="${1}" - local src="${GIT_PACKAGING_REPOS_URL}/${pkgbase}.git" - local target="${GIT_PACKAGES_CACHE}/${pkgbase}" + local project_path src target + project_path=$(gitlab_project_name_to_path "${pkgbase}") + src="${GIT_PACKAGING_REPOS_URL}/${project_path}.git" + target="${GIT_PACKAGES_CACHE}/${pkgbase}" if [[ ! -d $target ]]; then if ! arch_git -c core.sharedRepository=group clone --origin origin --bare --mirror "${src}" "${target}"; then return 1