feat(clone): speedup maintainer and universe clone query
There is a single endpoint now to list all pkgbases and their current maintainers. Use this endpoint for speeding up the clone of all packages of a maintainer by only issuing a single API call. Component: pkgctl repo clone Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
This commit is contained in:
parent
c356995dc1
commit
f2cafa3cb0
|
@ -0,0 +1,57 @@
|
||||||
|
#!/hint/bash
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
[[ -z ${DEVTOOLS_INCLUDE_API_ARCHWEB_SH:-} ]] || return 0
|
||||||
|
DEVTOOLS_INCLUDE_API_ARCHWEB_SH=1
|
||||||
|
|
||||||
|
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
|
||||||
|
# shellcheck source=src/lib/common.sh
|
||||||
|
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
|
||||||
|
archweb_query_all_packages() {
|
||||||
|
[[ -z ${WORKDIR:-} ]] && setup_workdir
|
||||||
|
|
||||||
|
stat_busy "Query all released packages"
|
||||||
|
mapfile -t pkgbases < <(
|
||||||
|
curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
|
||||||
|
"${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
|
||||||
|
| jq --raw-output --exit-status 'keys[]' 2> "${WORKDIR}/error"
|
||||||
|
)
|
||||||
|
if ! wait $!; then
|
||||||
|
stat_failed
|
||||||
|
print_workdir_error
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
stat_done
|
||||||
|
|
||||||
|
printf "%s\n" "${pkgbases[@]}"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
archweb_query_maintainer_packages() {
|
||||||
|
local maintainer=$1
|
||||||
|
|
||||||
|
[[ -z ${WORKDIR:-} ]] && setup_workdir
|
||||||
|
|
||||||
|
stat_busy "Query maintainer packages"
|
||||||
|
mapfile -t pkgbases < <(
|
||||||
|
curl --location --show-error --no-progress-meter --fail --retry 3 --retry-delay 3 \
|
||||||
|
"${PKGBASE_MAINTAINER_URL}" 2> "${WORKDIR}/error" \
|
||||||
|
| jq --raw-output --exit-status '. as $parent | keys[] | select(. as $key | $parent[$key] | index("'"${maintainer}"'"))' 2> "${WORKDIR}/error"
|
||||||
|
)
|
||||||
|
if ! wait $!; then
|
||||||
|
stat_failed
|
||||||
|
print_workdir_error
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
stat_done
|
||||||
|
|
||||||
|
printf "%s\n" "${pkgbases[@]}"
|
||||||
|
return 0
|
||||||
|
}
|
|
@ -28,6 +28,7 @@ export GIT_PACKAGING_NAMESPACE_ID=11323
|
||||||
export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}"
|
export GIT_PACKAGING_URL_SSH="git@${GITLAB_HOST}:${GIT_PACKAGING_NAMESPACE}"
|
||||||
export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE}"
|
export GIT_PACKAGING_URL_HTTPS="https://${GITLAB_HOST}/${GIT_PACKAGING_NAMESPACE}"
|
||||||
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
|
export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org
|
||||||
|
export PKGBASE_MAINTAINER_URL=https://archlinux.org/packages/pkgbase-maintainer
|
||||||
|
|
||||||
# check if messages are to be printed using color
|
# check if messages are to be printed using color
|
||||||
if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
|
if [[ -t 2 && "$TERM" != dumb ]] || [[ ${DEVTOOLS_COLOR} == always ]]; then
|
||||||
|
@ -53,6 +54,11 @@ stat_done() {
|
||||||
printf "${BOLD}done${ALL_OFF}\n" >&2
|
printf "${BOLD}done${ALL_OFF}\n" >&2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
stat_failed() {
|
||||||
|
# shellcheck disable=2059
|
||||||
|
printf "${BOLD}${RED}failed${ALL_OFF}\n" >&2
|
||||||
|
}
|
||||||
|
|
||||||
msg_success() {
|
msg_success() {
|
||||||
local msg=$1
|
local msg=$1
|
||||||
local padding
|
local padding
|
||||||
|
@ -77,6 +83,15 @@ msg_warn() {
|
||||||
printf "%s %s\n" "${padding}${YELLOW}!${ALL_OFF}" "${msg}" >&2
|
printf "%s %s\n" "${padding}${YELLOW}!${ALL_OFF}" "${msg}" >&2
|
||||||
}
|
}
|
||||||
|
|
||||||
|
print_workdir_error() {
|
||||||
|
if [[ ! -f "${WORKDIR}"/error ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
while read -r LINE; do
|
||||||
|
error '%s' "${LINE}"
|
||||||
|
done < "${WORKDIR}/error"
|
||||||
|
}
|
||||||
|
|
||||||
_setup_workdir=false
|
_setup_workdir=false
|
||||||
setup_workdir() {
|
setup_workdir() {
|
||||||
[[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX")
|
[[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX")
|
||||||
|
|
|
@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_REPO_CLONE_SH=1
|
||||||
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
|
_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
|
||||||
# shellcheck source=src/lib/common.sh
|
# shellcheck source=src/lib/common.sh
|
||||||
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
|
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
|
||||||
|
# shellcheck source=src/lib/api/archweb.sh
|
||||||
|
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/archweb.sh
|
||||||
# shellcheck source=src/lib/api/gitlab.sh
|
# shellcheck source=src/lib/api/gitlab.sh
|
||||||
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh
|
source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh
|
||||||
# shellcheck source=src/lib/repo/configure.sh
|
# shellcheck source=src/lib/repo/configure.sh
|
||||||
|
@ -18,6 +20,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
|
||||||
source /usr/share/makepkg/util/message.sh
|
source /usr/share/makepkg/util/message.sh
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
set -o pipefail
|
||||||
|
|
||||||
|
|
||||||
pkgctl_repo_clone_usage() {
|
pkgctl_repo_clone_usage() {
|
||||||
|
@ -137,33 +140,18 @@ pkgctl_repo_clone() {
|
||||||
|
|
||||||
# Query packages of a maintainer
|
# Query packages of a maintainer
|
||||||
if [[ -n ${MAINTAINER} ]]; then
|
if [[ -n ${MAINTAINER} ]]; then
|
||||||
stat_busy "Query packages"
|
mapfile -t pkgbases < <(archweb_query_maintainer_packages "${MAINTAINER}")
|
||||||
max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages')
|
if ! wait $!; then
|
||||||
if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
|
die "Failed to query maintainer packages"
|
||||||
stat_done
|
|
||||||
warning "found no packages for maintainer ${MAINTAINER}"
|
|
||||||
exit 0
|
|
||||||
fi
|
fi
|
||||||
mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
|
|
||||||
curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase'
|
|
||||||
stat_progress
|
|
||||||
done | sort --unique)
|
|
||||||
stat_done
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Query all released packages
|
# Query all released packages
|
||||||
if (( CLONE_ALL )); then
|
if (( CLONE_ALL )); then
|
||||||
stat_busy "Query all released packages"
|
mapfile -t pkgbases < <(archweb_query_all_packages)
|
||||||
max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages')
|
if ! wait $!; then
|
||||||
if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
|
die "Failed to query all packages"
|
||||||
stat_done
|
|
||||||
die "failed to query packages"
|
|
||||||
fi
|
fi
|
||||||
mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
|
|
||||||
curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase'
|
|
||||||
stat_progress
|
|
||||||
done | sort --unique)
|
|
||||||
stat_done
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# parallelization
|
# parallelization
|
||||||
|
|
|
@ -39,6 +39,8 @@ fi
|
||||||
|
|
||||||
export _DEVTOOLS_COMMAND='pkgctl'
|
export _DEVTOOLS_COMMAND='pkgctl'
|
||||||
|
|
||||||
|
setup_workdir
|
||||||
|
|
||||||
load_devtools_config
|
load_devtools_config
|
||||||
|
|
||||||
# command checking
|
# command checking
|
||||||
|
|
Loading…
Reference in New Issue