diff --git a/tests/kernel-ark-build/runtest.sh b/tests/kernel-ark-build/runtest.sh index 7342048..f9b217f 100755 --- a/tests/kernel-ark-build/runtest.sh +++ b/tests/kernel-ark-build/runtest.sh @@ -2,11 +2,17 @@ # vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k . /usr/share/beakerlib/beakerlib.sh || exit 1 -## General configuration variables. Can be overriden via environment +## General configuration variables. Can be overridden via environment # Kernel branch to build. Typically ark-latest or os-build. KERNEL_BRANCH=${KERNEL_BRANCH:-"ark-latest"} KERNEL_GIT_URL=${KERNEL_GIT_URL:-"https://gitlab.com/cki-project/kernel-ark.git"} +## Helper variables to handle cached/generated SRPM +CACHE_DIR_BASE="/var/tmp/kernel-" +KERNEL_SRPM_PATTERN="kernel-*.src.rpm" +# Path to the kernel SRPM to build. +KERNEL_SRPM="" + # Environment variables to configure the kernel build: # * ENABLE_LTO: If set, it will build kernel-ark with clang_lto. # * ENABLE_DEBUG: If set, it will build kernel-ark in debug mode. @@ -16,8 +22,8 @@ CLANG_MODE=${CLANG_MODE:-"--with clang"} BUILD_MODE=${ENABLE_DEBUG:+"--with debug --without base"} BUILD_MODE=${BUILD_MODE:-"--with base --without debug"} -function logTmtRequiredPackages(){ - # Get the expected packages from TMT metadata requires +# Log installed packages required by TMT +logTmtRequiredPackages() { if [[ ! -e $TMT_TEST_METADATA ]]; then rlLogWarning "${FUNCNAME[0]}: No TMT_TEST_METADATA file found. Run the test from tmt" return @@ -26,15 +32,109 @@ function logTmtRequiredPackages(){ rlLog "# Installed packages:" rlLog "#----------------------------#" for pkg in $TMT_REQUIRES; do - rlAssertRpm $pkg - done; + rlAssertRpm "$pkg" + done rlLog "#----------------------------#" +} +# Check if a previous test run created a kernel SRPM for the latest commit +# If found, the abspath is stored in KERNEL_SRPM global variable +# Returns 0 if srpm found, 1 otherwise. +checkCache() { + local commit_hash + local cache_dir + local cached_srpm + + commit_hash=$(git ls-remote "${KERNEL_GIT_URL}" "${KERNEL_BRANCH}" | awk '{print $1}') + rlLog "Latest commit for ${KERNEL_BRANCH} branch: $commit_hash" + + cache_dir="${CACHE_DIR_BASE}${commit_hash}/" + if [[ -d "$cache_dir" ]]; then + cached_srpm=$(find "$cache_dir" -name "${KERNEL_SRPM_PATTERN}") + if [[ -n "$cached_srpm" ]]; then + rlLog "Found cached SRPM: $cached_srpm" + rlRun "dnf builddep -y $cached_srpm > install-buildreqs.log 2>&1" + rlFileSubmit install-buildreqs.log + KERNEL_SRPM="$cached_srpm" + return 0 + fi + fi + rlLog "Cached kernel SRPM not found" + return 1 +} + +# Attempts to clone the kernel-ark tree in the CWD, at most 3 times. +# Returns 0 on success, 1 otherwise +cloneKernelTree() { + local retries=3 + local delay=10 + + rlLog "Cloning kernel" + while [[ $retries -gt 0 ]]; do + rlRun "git clone -q --branch ${KERNEL_BRANCH} ${KERNEL_GIT_URL}" 0-255 + retcode=$? + if [[ $retcode -eq 0 ]]; then + return 0 + else + retries=$((retries - 1)) + rlLog "Clone failed, wait ${delay}s and retry ($retries retries left)" + sleep $delay + fi + done + return $retcode +} + +# In a kernel-ark repository, do all the operations needed to generate an SRPM +# If the kernel is generated return the abspath in KERNEL_SRPM global variable +# Returns 0 on success, 1 otherwise +generateSRPM() { + local generated_srpm + local commit_hash + local cache_dir + + if ! cloneKernelTree; then + rlFail "Failed to clone kernel tree" + return 1 + fi + + rlRun "pushd kernel-ark" + rlLog "Gathering and installing missing build requirements" + rlRun "make dist-get-buildreqs > make-buildreqs.log 2>&1" + rlFileSubmit make-buildreqs.log + if grep -q 'Missing dependencies:' make-buildreqs.log; then + rlRun "KERNEL_BUILDREQS=\"$(sed -n 's/Missing dependencies://p' make-buildreqs.log)\"" + rlRun 'dnf install -y ${KERNEL_BUILDREQS} > install-buildreqs.log 2>&1' + rlFileSubmit install-buildreqs.log + elif grep 'PASS:' make-buildreqs.log; then + rlLog "All dependencies were already installed" + else + rlLogWarning "Error getting dependencies, the build might fail" + rlRun "cat make-buildreqs.log" + fi + + rlLog "Generating SRPM" + rlRun "make dist-srpm > dist-srpm.log 2>&1" + rlFileSubmit dist-srpm.log + + generated_srpm=$(find "$(pwd)" -name "$KERNEL_SRPM_PATTERN") + if [[ -e "$generated_srpm" ]]; then + rlLog "Caching generated SRPM" + commit_hash=$(git rev-parse HEAD) + cache_dir="${CACHE_DIR_BASE}${commit_hash}/" + rlRun "mkdir -p $cache_dir" + rlRun "cp $generated_srpm $cache_dir" + KERNEL_SRPM="$generated_srpm" + rlRun "popd" + return 0 + else + rlLogWarning "Failed to generate SRPM" + rlRun "popd" + return 1 + fi } rlJournalStart rlPhaseStartSetup - # Log system mem/cpus and packages installed by tmt rlRun "nproc" rlRun "free -h" logTmtRequiredPackages @@ -49,42 +149,28 @@ rlJournalStart rlPhaseEnd rlPhaseStartTest - # TODO It would be fantastic to find out a way to shallow clone without - # breaking the dist-get-buildreqs and dist-srpm make targets, because - # a complete clone takes about 5GiB and a LOT of time... - rlLog "Cloning kernel" - rlRun "git clone --branch ${KERNEL_BRANCH} ${KERNEL_GIT_URL}" - rlRun 'cd kernel-ark' - - rlLog "Gathering and installing missing build requirements" - rlRun "make dist-get-buildreqs > make-buildreqs.log" - rlFileSubmit make-buildreqs.log - if grep 'Missing dependencies:' make-buildreqs.log; then - rlRun "KERNEL_BUILDREQS=\"$(sed -n 's/Missing dependencies://p' make-buildreqs.log)\"" - rlRun 'dnf install -y ${KERNEL_BUILDREQS} > install-buildreqs.log 2>&1' - rlFileSubmit install-buildreqs.log - elif grep 'PASS:' make-buildreqs.log; then - rlLog "All dependencies were already installed" + # Check if kernel package was generated by a previous test run + # and generate a new one if it wasn't + checkCache + if [[ -e "$KERNEL_SRPM" ]]; then + rlLog "Using cached kernel SRPM: $KERNEL_SRPM" else - rlLogWarning "Error getting dependencies, the build might fail" - rlRun "cat make-buildreqs.log" + rlLog "No kernel SRPM found, generating new SRPM" + generateSRPM fi - rlLog "Generating srpm" - rlRun 'make dist-srpm > dist-srpm.log 2>&1' - rlRun 'KERNEL_SRPM=$(find . -name kernel-*.src.rpm)' - rlFileSubmit dist-srpm.log - - # Prepare the flags to be passed to rpmbuild depending on the configuration - BUILD_FLAGS="--target $(uname -m) --with up --with toolchain_clang " - BUILD_FLAGS+="--without trace --without arm64_16k --without arm64_64k " - BUILD_FLAGS+="--without realtime --without zfcpdump " - BUILD_FLAGS+="${CLANG_MODE} ${BUILD_MODE}" - - rlLog "Building kernel (branch $KERNEL_BRANCH)" - rlRun "rpmbuild ${BUILD_FLAGS} --rebuild ${KERNEL_SRPM} > build.log 2>&1" - rlFileSubmit build.log + if [[ -e "$KERNEL_SRPM" ]]; then + BUILD_FLAGS="--target $(uname -m) --with up --with toolchain_clang " + BUILD_FLAGS+="--without trace --without arm64_16k --without arm64_64k " + BUILD_FLAGS+="--without realtime --without zfcpdump " + BUILD_FLAGS+="${CLANG_MODE} ${BUILD_MODE}" + rlLog "Building kernel (branch $KERNEL_BRANCH)" + rlRun "rpmbuild ${BUILD_FLAGS} --rebuild ${KERNEL_SRPM} > build.log 2>&1" + rlFileSubmit build.log + else + rlFail "Kernel SRPM not found, finishing test" + fi rlPhaseEnd rlPhaseStartCleanup