Refactor kernel-ark build test for caching and git clone retry
This commit mitigates test failures due to git unreliability and improves test time over test reruns using cached SRPMs. - Added CACHE_DIR_BASE, KERNEL_SRPM_PATTERN and KERNEL SRPM variables. - Implemented checkCache() to look for an existing SRPM in the cache and install its build dependencies. - Implemented cloneKernelTree() for cloning the kernel-ark repo with retries. - Implemented generateSRPM() to handle all operations required to generate an SRPM and cache it. - Updated main test phase to utilize caching and SRPM generation functions.
This commit is contained in:
Jesus Checa Hidalgo
jcheca
parent
bf7e2faa36
commit
71a2b1c3bf
1 changed files with 124 additions and 38 deletions
|
|
@ -2,11 +2,17 @@
|
|||
# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
|
||||
. /usr/share/beakerlib/beakerlib.sh || exit 1
|
||||
|
||||
## General configuration variables. Can be overriden via environment
|
||||
## General configuration variables. Can be overridden via environment
|
||||
# Kernel branch to build. Typically ark-latest or os-build.
|
||||
KERNEL_BRANCH=${KERNEL_BRANCH:-"ark-latest"}
|
||||
KERNEL_GIT_URL=${KERNEL_GIT_URL:-"https://gitlab.com/cki-project/kernel-ark.git"}
|
||||
|
||||
## Helper variables to handle cached/generated SRPM
|
||||
CACHE_DIR_BASE="/var/tmp/kernel-"
|
||||
KERNEL_SRPM_PATTERN="kernel-*.src.rpm"
|
||||
# Path to the kernel SRPM to build.
|
||||
KERNEL_SRPM=""
|
||||
|
||||
# Environment variables to configure the kernel build:
|
||||
# * ENABLE_LTO: If set, it will build kernel-ark with clang_lto.
|
||||
# * ENABLE_DEBUG: If set, it will build kernel-ark in debug mode.
|
||||
|
|
@ -16,8 +22,8 @@ CLANG_MODE=${CLANG_MODE:-"--with clang"}
|
|||
BUILD_MODE=${ENABLE_DEBUG:+"--with debug --without base"}
|
||||
BUILD_MODE=${BUILD_MODE:-"--with base --without debug"}
|
||||
|
||||
function logTmtRequiredPackages(){
|
||||
# Get the expected packages from TMT metadata requires
|
||||
# Log installed packages required by TMT
|
||||
logTmtRequiredPackages() {
|
||||
if [[ ! -e $TMT_TEST_METADATA ]]; then
|
||||
rlLogWarning "${FUNCNAME[0]}: No TMT_TEST_METADATA file found. Run the test from tmt"
|
||||
return
|
||||
|
|
@ -26,15 +32,109 @@ function logTmtRequiredPackages(){
|
|||
rlLog "# Installed packages:"
|
||||
rlLog "#----------------------------#"
|
||||
for pkg in $TMT_REQUIRES; do
|
||||
rlAssertRpm $pkg
|
||||
done;
|
||||
rlAssertRpm "$pkg"
|
||||
done
|
||||
rlLog "#----------------------------#"
|
||||
}
|
||||
|
||||
# Check if a previous test run created a kernel SRPM for the latest commit
|
||||
# If found, the abspath is stored in KERNEL_SRPM global variable
|
||||
# Returns 0 if srpm found, 1 otherwise.
|
||||
checkCache() {
|
||||
local commit_hash
|
||||
local cache_dir
|
||||
local cached_srpm
|
||||
|
||||
commit_hash=$(git ls-remote "${KERNEL_GIT_URL}" "${KERNEL_BRANCH}" | awk '{print $1}')
|
||||
rlLog "Latest commit for ${KERNEL_BRANCH} branch: $commit_hash"
|
||||
|
||||
cache_dir="${CACHE_DIR_BASE}${commit_hash}/"
|
||||
if [[ -d "$cache_dir" ]]; then
|
||||
cached_srpm=$(find "$cache_dir" -name "${KERNEL_SRPM_PATTERN}")
|
||||
if [[ -n "$cached_srpm" ]]; then
|
||||
rlLog "Found cached SRPM: $cached_srpm"
|
||||
rlRun "dnf builddep -y $cached_srpm > install-buildreqs.log 2>&1"
|
||||
rlFileSubmit install-buildreqs.log
|
||||
KERNEL_SRPM="$cached_srpm"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
rlLog "Cached kernel SRPM not found"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Attempts to clone the kernel-ark tree in the CWD, at most 3 times.
|
||||
# Returns 0 on success, 1 otherwise
|
||||
cloneKernelTree() {
|
||||
local retries=3
|
||||
local delay=10
|
||||
|
||||
rlLog "Cloning kernel"
|
||||
while [[ $retries -gt 0 ]]; do
|
||||
rlRun "git clone -q --branch ${KERNEL_BRANCH} ${KERNEL_GIT_URL}" 0-255
|
||||
retcode=$?
|
||||
if [[ $retcode -eq 0 ]]; then
|
||||
return 0
|
||||
else
|
||||
retries=$((retries - 1))
|
||||
rlLog "Clone failed, wait ${delay}s and retry ($retries retries left)"
|
||||
sleep $delay
|
||||
fi
|
||||
done
|
||||
return $retcode
|
||||
}
|
||||
|
||||
# In a kernel-ark repository, do all the operations needed to generate an SRPM
|
||||
# If the kernel is generated return the abspath in KERNEL_SRPM global variable
|
||||
# Returns 0 on success, 1 otherwise
|
||||
generateSRPM() {
|
||||
local generated_srpm
|
||||
local commit_hash
|
||||
local cache_dir
|
||||
|
||||
if ! cloneKernelTree; then
|
||||
rlFail "Failed to clone kernel tree"
|
||||
return 1
|
||||
fi
|
||||
|
||||
rlRun "pushd kernel-ark"
|
||||
rlLog "Gathering and installing missing build requirements"
|
||||
rlRun "make dist-get-buildreqs > make-buildreqs.log 2>&1"
|
||||
rlFileSubmit make-buildreqs.log
|
||||
if grep -q 'Missing dependencies:' make-buildreqs.log; then
|
||||
rlRun "KERNEL_BUILDREQS=\"$(sed -n 's/Missing dependencies://p' make-buildreqs.log)\""
|
||||
rlRun 'dnf install -y ${KERNEL_BUILDREQS} > install-buildreqs.log 2>&1'
|
||||
rlFileSubmit install-buildreqs.log
|
||||
elif grep 'PASS:' make-buildreqs.log; then
|
||||
rlLog "All dependencies were already installed"
|
||||
else
|
||||
rlLogWarning "Error getting dependencies, the build might fail"
|
||||
rlRun "cat make-buildreqs.log"
|
||||
fi
|
||||
|
||||
rlLog "Generating SRPM"
|
||||
rlRun "make dist-srpm > dist-srpm.log 2>&1"
|
||||
rlFileSubmit dist-srpm.log
|
||||
|
||||
generated_srpm=$(find "$(pwd)" -name "$KERNEL_SRPM_PATTERN")
|
||||
if [[ -e "$generated_srpm" ]]; then
|
||||
rlLog "Caching generated SRPM"
|
||||
commit_hash=$(git rev-parse HEAD)
|
||||
cache_dir="${CACHE_DIR_BASE}${commit_hash}/"
|
||||
rlRun "mkdir -p $cache_dir"
|
||||
rlRun "cp $generated_srpm $cache_dir"
|
||||
KERNEL_SRPM="$generated_srpm"
|
||||
rlRun "popd"
|
||||
return 0
|
||||
else
|
||||
rlLogWarning "Failed to generate SRPM"
|
||||
rlRun "popd"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
rlJournalStart
|
||||
rlPhaseStartSetup
|
||||
# Log system mem/cpus and packages installed by tmt
|
||||
rlRun "nproc"
|
||||
rlRun "free -h"
|
||||
logTmtRequiredPackages
|
||||
|
|
@ -49,42 +149,28 @@ rlJournalStart
|
|||
rlPhaseEnd
|
||||
|
||||
rlPhaseStartTest
|
||||
# TODO It would be fantastic to find out a way to shallow clone without
|
||||
# breaking the dist-get-buildreqs and dist-srpm make targets, because
|
||||
# a complete clone takes about 5GiB and a LOT of time...
|
||||
rlLog "Cloning kernel"
|
||||
rlRun "git clone --branch ${KERNEL_BRANCH} ${KERNEL_GIT_URL}"
|
||||
rlRun 'cd kernel-ark'
|
||||
|
||||
rlLog "Gathering and installing missing build requirements"
|
||||
rlRun "make dist-get-buildreqs > make-buildreqs.log"
|
||||
rlFileSubmit make-buildreqs.log
|
||||
if grep 'Missing dependencies:' make-buildreqs.log; then
|
||||
rlRun "KERNEL_BUILDREQS=\"$(sed -n 's/Missing dependencies://p' make-buildreqs.log)\""
|
||||
rlRun 'dnf install -y ${KERNEL_BUILDREQS} > install-buildreqs.log 2>&1'
|
||||
rlFileSubmit install-buildreqs.log
|
||||
elif grep 'PASS:' make-buildreqs.log; then
|
||||
rlLog "All dependencies were already installed"
|
||||
# Check if kernel package was generated by a previous test run
|
||||
# and generate a new one if it wasn't
|
||||
checkCache
|
||||
if [[ -e "$KERNEL_SRPM" ]]; then
|
||||
rlLog "Using cached kernel SRPM: $KERNEL_SRPM"
|
||||
else
|
||||
rlLogWarning "Error getting dependencies, the build might fail"
|
||||
rlRun "cat make-buildreqs.log"
|
||||
rlLog "No kernel SRPM found, generating new SRPM"
|
||||
generateSRPM
|
||||
fi
|
||||
|
||||
rlLog "Generating srpm"
|
||||
rlRun 'make dist-srpm > dist-srpm.log 2>&1'
|
||||
rlRun 'KERNEL_SRPM=$(find . -name kernel-*.src.rpm)'
|
||||
rlFileSubmit dist-srpm.log
|
||||
|
||||
# Prepare the flags to be passed to rpmbuild depending on the configuration
|
||||
BUILD_FLAGS="--target $(uname -m) --with up --with toolchain_clang "
|
||||
BUILD_FLAGS+="--without trace --without arm64_16k --without arm64_64k "
|
||||
BUILD_FLAGS+="--without realtime --without zfcpdump "
|
||||
BUILD_FLAGS+="${CLANG_MODE} ${BUILD_MODE}"
|
||||
|
||||
rlLog "Building kernel (branch $KERNEL_BRANCH)"
|
||||
rlRun "rpmbuild ${BUILD_FLAGS} --rebuild ${KERNEL_SRPM} > build.log 2>&1"
|
||||
rlFileSubmit build.log
|
||||
if [[ -e "$KERNEL_SRPM" ]]; then
|
||||
BUILD_FLAGS="--target $(uname -m) --with up --with toolchain_clang "
|
||||
BUILD_FLAGS+="--without trace --without arm64_16k --without arm64_64k "
|
||||
BUILD_FLAGS+="--without realtime --without zfcpdump "
|
||||
BUILD_FLAGS+="${CLANG_MODE} ${BUILD_MODE}"
|
||||
|
||||
rlLog "Building kernel (branch $KERNEL_BRANCH)"
|
||||
rlRun "rpmbuild ${BUILD_FLAGS} --rebuild ${KERNEL_SRPM} > build.log 2>&1"
|
||||
rlFileSubmit build.log
|
||||
else
|
||||
rlFail "Kernel SRPM not found, finishing test"
|
||||
fi
|
||||
rlPhaseEnd
|
||||
|
||||
rlPhaseStartCleanup
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue