From 4dcf94c473a0b1c3ba03a61b852e35cbb4f08d7c Mon Sep 17 00:00:00 2001 From: Edjunior Machado Date: Tue, 27 Apr 2021 20:17:24 +0200 Subject: [PATCH] Introduce CI gating --- .fmf/version | 1 + gating.yaml | 19 +++ plans/ci.fmf | 6 + tests/Regression/upstream-testsuite/Makefile | 64 ++++++++++ tests/Regression/upstream-testsuite/PURPOSE | 3 + tests/Regression/upstream-testsuite/main.fmf | 25 ++++ .../Regression/upstream-testsuite/runtest.sh | 113 ++++++++++++++++++ 7 files changed, 231 insertions(+) create mode 100644 .fmf/version create mode 100644 gating.yaml create mode 100644 plans/ci.fmf create mode 100644 tests/Regression/upstream-testsuite/Makefile create mode 100644 tests/Regression/upstream-testsuite/PURPOSE create mode 100644 tests/Regression/upstream-testsuite/main.fmf create mode 100755 tests/Regression/upstream-testsuite/runtest.sh diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/gating.yaml b/gating.yaml new file mode 100644 index 0000000..ce3cdc1 --- /dev/null +++ b/gating.yaml @@ -0,0 +1,19 @@ +--- !Policy +product_versions: + - fedora-* +decision_context: bodhi_update_push_stable +subject_type: koji_build +rules: + - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional} +--- !Policy +product_versions: + - rhel-8 +decision_context: osci_compose_gate +rules: + - !PassingTestCaseRule {test_case_name: baseos-ci.brew-build.tier1.functional} +--- !Policy +product_versions: + - rhel-9 +decision_context: osci_compose_gate +rules: + - !PassingTestCaseRule {test_case_name: baseos-ci.brew-build.tier1.functional} diff --git a/plans/ci.fmf b/plans/ci.fmf new file mode 100644 index 0000000..1ad2c12 --- /dev/null +++ b/plans/ci.fmf @@ -0,0 +1,6 @@ +summary: CI Gating Plan +discover: + how: fmf + directory: tests +execute: + how: beakerlib diff --git a/tests/Regression/upstream-testsuite/Makefile b/tests/Regression/upstream-testsuite/Makefile new file mode 100644 index 0000000..dbe57e6 --- /dev/null +++ b/tests/Regression/upstream-testsuite/Makefile @@ -0,0 +1,64 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Makefile of /tools/cppunit/Regression/upstream-testsuite +# Description: Runs the upstream cppunit testsuite. +# Author: Arjun Shankar +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +export TEST=/tools/cppunit/Regression/upstream-testsuite +export TESTVERSION=1.0 + +BUILT_FILES= + +FILES=$(METADATA) runtest.sh Makefile PURPOSE + +.PHONY: all install download clean + +run: $(FILES) build + ./runtest.sh + +build: $(BUILT_FILES) + test -x runtest.sh || chmod a+x runtest.sh + +clean: + rm -f *~ $(BUILT_FILES) + + +include /usr/share/rhts/lib/rhts-make.include + +$(METADATA): Makefile + @echo "Owner: Arjun Shankar " > $(METADATA) + @echo "Name: $(TEST)" >> $(METADATA) + @echo "TestVersion: $(TESTVERSION)" >> $(METADATA) + @echo "Path: $(TEST_DIR)" >> $(METADATA) + @echo "Description: Runs the upstream cppunit testsuite." >> $(METADATA) + @echo "Type: Regression" >> $(METADATA) + @echo "TestTime: 30m" >> $(METADATA) + @echo "RunFor: cppunit" >> $(METADATA) + @echo "Requires: automake cppunit doxygen gcc-c++ glibc-devel glibc-headers graphviz libstdc++ libstdc++-devel libstdc++-static make rpm-build" >> $(METADATA) + @echo "Requires: pkgconfig pkgconf-pkg-config" >> $(METADATA) + @echo "Priority: Normal" >> $(METADATA) + @echo "License: GPLv2+" >> $(METADATA) + @echo "Confidential: no" >> $(METADATA) + @echo "Destructive: no" >> $(METADATA) + @echo "Releases: -RHEL4 -RHELClient5 -RHELServer5" >> $(METADATA) + + rhts-lint $(METADATA) diff --git a/tests/Regression/upstream-testsuite/PURPOSE b/tests/Regression/upstream-testsuite/PURPOSE new file mode 100644 index 0000000..5d783ba --- /dev/null +++ b/tests/Regression/upstream-testsuite/PURPOSE @@ -0,0 +1,3 @@ +PURPOSE of /tools/cppunit/Regression/upstream-testsuite +Description: Runs the upstream cppunit testsuite. +Author: Arjun Shankar diff --git a/tests/Regression/upstream-testsuite/main.fmf b/tests/Regression/upstream-testsuite/main.fmf new file mode 100644 index 0000000..4932247 --- /dev/null +++ b/tests/Regression/upstream-testsuite/main.fmf @@ -0,0 +1,25 @@ +summary: Runs the upstream cppunit testsuite. +description: '' +contact: Arjun Shankar +component: +- cppunit +test: ./runtest.sh +framework: beakerlib +recommend: +- automake +- cppunit +- doxygen +- gcc-c++ +- glibc-devel +- glibc-headers +- graphviz +- libstdc++ +- libstdc++-devel +- libstdc++-static +- make +- rpm-build +- pkgconfig +- pkgconf-pkg-config +duration: 30m +extra-summary: /tools/cppunit/Regression/upstream-testsuite +extra-task: /tools/cppunit/Regression/upstream-testsuite diff --git a/tests/Regression/upstream-testsuite/runtest.sh b/tests/Regression/upstream-testsuite/runtest.sh new file mode 100755 index 0000000..89b4d67 --- /dev/null +++ b/tests/Regression/upstream-testsuite/runtest.sh @@ -0,0 +1,113 @@ +#!/bin/bash +# vim: dict+=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# runtest.sh of /tools/cppunit/Regression/upstream-testsuite +# Description: Runs the upstream cppunit testsuite. +# Author: Arjun Shankar +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This program is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 2 of +# the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied +# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +# PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# Include Beaker environment +. /usr/share/beakerlib/beakerlib.sh || exit 1 + +PACKAGE="cppunit" + +# On RHEL-8, it requires CRB repo enabled for doxygen +REQUIRES="automake cppunit doxygen gcc-c++ glibc-devel graphviz libstdc++ libstdc++-devel make rpm-build" + +rlJournalStart + rlPhaseStartSetup + rlAssertRpm --all + rlCheckRpm libstdc++-static + rlCheckRpm glibc-headers + rlCheckRpm pkgconfig + rlCheckRpm pkgconf-pkg-config + + rlRun "TmpDir=\$(mktemp -d)" 0 "Creating tmp directory" + rlRun "pushd $TmpDir" + + # Get rid of any existing builds + BUILDDIR="$(rpm -E '%{_builddir}')" + SPECDIR="$(rpm -E '%{_specdir}')" + rlRun "rm -rf $BUILDDIR/cppunit*" + rlRun "rm -rf $SPECDIR/cppunit*.spec" + + # Download cppunit srpm into current directory + rlFetchSrcForInstalled cppunit || yumdownloader --source cppunit + rlRun "ls cppunit*.src.rpm" # rlAssertExists doesn't accept wildcards + + # Extract into /root/rpmbuild/ + rlRun "rpm -Uv --replacepkgs cppunit*.src.rpm" + rlAssertExists $SPECDIR/cppunit.spec + + rlPhaseEnd + + rlPhaseStartTest + CPPUNIT_BUILD_LOG=$(pwd)/cppunit-rpmbuild.log + + rlRun "rpmbuild -bc $SPECDIR/cppunit.spec --target=$(arch) &> $CPPUNIT_BUILD_LOG" + rlFileSubmit $CPPUNIT_BUILD_LOG "cppunit-rpmbuild.log" + + rlRun "ls $BUILDDIR | grep cppunit" + BUILD_SUBDIR="$BUILDDIR/$(ls $BUILDDIR | grep cppunit)" + rlRun "pushd $BUILD_SUBDIR" + + CHECK_LOG=$(pwd)/check.log + rlRun "make -k check &> $CHECK_LOG" + + if grep -q "^Testsuite summary for cppunit" $CHECK_LOG; then + rlRun -l "sed -n '/^Testsuite summary for cppunit/,/^# ERROR/{p;n;p}' $CHECK_LOG" + else + rlRun -l "sed -n '/^==================/,/^==================/p' $CHECK_LOG" + fi + + rlFileSubmit $CHECK_LOG "cppunit-check.log" + + # Verify expected PASS/FAILs: + if rlIsRHEL "<=7"; then + rlAssertEquals "Exactly 1 line of output corresponding to 2 failed tests" "$(cat $CHECK_LOG | grep 'Run:\s*5\s*Failures:\s*2\s*Errors:\s*0' | wc -l)" "1" + rlAssertEquals "Exactly 1 line of output '!!!FAILURES!!!'" "$(cat $CHECK_LOG | grep '^!!!FAILURES!!!' | wc -l)" "1" + rlAssertEquals "Exactly 1 line of output 'XFAIL'" "$(cat $CHECK_LOG | grep '^XFAIL' | wc -l)" "1" + rlAssertGreaterOrEqual "At least 222 OK tests" "$(cat $CHECK_LOG | grep ': OK$' | wc -l)" "222" + else + awk 'BEGIN {p=0; f=0; e=0}; /^# PASS:/ {p+=$3}; /^# FAIL:/ {f+=$3}; /^# ERROR:/ {e+=$3}; END {print (p>0?1:0)" "f" "e}' $CHECK_LOG | fgrep -q '1 0 0' + retval=$? + rlRun "test $retval -eq 0" 0 "Examine the test log" + fi + + rlRun "popd" + rlPhaseEnd + + rlPhaseStartCleanup + if [ -n "$KEEP_CPPUNIT_RESULTS" ]; then + rlLog "Build Directory at: $(ls $BUILDDIR)" + rlLog "Spec File at: $(ls $SPECDIR/cppunit*.spec)" + rlRun "popd" + else + rlRun "rm cppunit*.src.rpm" + rlRun "rm -rf $BUILDDIR/cppunit* $SPECDIR/cppunit*.spec" + rlRun "ls" + rlRun "popd" + rlRun "rm -r $TmpDir" + fi + rlPhaseEnd +rlJournalPrintText +rlJournalEnd