Sanity/smoke-test: new smoke test which runs upstream test suite

Sanity/smoke-test/fix_hugepages_err.patch

@@ -0,0 +1,12 @@
+diff --git a/tests/probes/sysctl/test_sysctl_probe_all.sh b/tests/probes/sysctl/test_sysctl_probe_all.sh
+index 732d22ec1..119e92c33 100755
+--- a/tests/probes/sysctl/test_sysctl_probe_all.sh
++++ b/tests/probes/sysctl/test_sysctl_probe_all.sh
+@@ -69,6 +69,7 @@ diff "$sysctlNames" "$ourNames"
+ 
+ # remove oscap error message related to permissions from stderr
+ sed -i -E "/^E: lt-probe_sysctl: Can't read sysctl value from /d" "$stderr"
++sed -i "/^.*hugepages.*$/d" "$stderr"
+ [ ! -s $stderr ]
+ 
+ rm $stderr $result $ourNames $sysctlNames

Sanity/smoke-test/main.fmf

@@ -0,0 +1,49 @@
+summary: Test runs upstream test suite.
+description: |
+    Runs upstream test suite against installed openscap. Test suite is built from
+    the openscap source RPM by running %prep and %build stages from the spec file.
+contact: Matus Marhefka <mmarhefk@redhat.com>
+component:
+- openscap
+test: ./runtest.sh
+framework: beakerlib
+require:
+- library(distribution/Cleanup)
+- library(distribution/RpmSnapshot)
+recommend:
+- openscap-scanner
+- openscap
+- openscap-engine-sce
+- python
+- python3
+- gcc
+- gcc-c++
+- perl-XML-XPath
+- yum-utils
+- rpm-build
+- dnf-plugins-core
+- wget
+- lua
+- which
+- procps-ng
+- initscripts
+- chkconfig
+- sendmail
+- bzip2
+- strace
+duration: 3h
+enabled: true
+tag:
+- CI-Tier-1
+- Fedora-Private
+- NoFIPS
+- Tier1
+- Tier1security
+- rhel8-buildroot
+- smoke
+tier: '1'
+relevancy: |
+    distro < rhel-5: False
+extra-summary: /CoreOS/openscap/Sanity/smoke-test
+extra-task: /CoreOS/openscap/Sanity/smoke-test
+extra-nitrate: TC#0075476

Sanity/smoke-test/review_notes

@@ -0,0 +1,16 @@
+This document is meant to help review process of openscap upstream test suite
+while doing rebases. At the very least the last reviewed version should be
+mentioned here. Plus any mandatory items with links to appropriate bugs, so
+changes (even innocent looking ones) can be evalued according to the real impact
+they might have to our productization efforts.
+
+For more detail, see https://tcms.engineering.redhat.com/case/571216
+
+To obtain a diff between commits/tags use:
+$ git log -p REVIEWED_COMMIT^..LATEST_COMMIT -- tests/
+To obtain only names of changed files add `--name-only`.
+Example:
+$ git log -p 1.3.1^..1.3.2 -- tests/
+
+Last reviewed version: openscap-1.3.3-1
+https://github.com/OpenSCAP/openscap/commit/0cb55c55af6be9934d6fd0caf4563b206f289732

Sanity/smoke-test/runtest.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+# vim: dict=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+#   runtest.sh of /CoreOS/openscap/Sanity/smoke-test
+#   Description: Test runs upstream test suite.
+#   Author: Ondrej Moris <omoris@redhat.com>
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+#
+#   Copyright (c) 2010 Red Hat, Inc. All rights reserved.
+#
+#   This copyrighted material is made available to anyone wishing
+#   to use, modify, copy, or redistribute it subject to the terms
+#   and conditions of the GNU General Public License version 2.
+#
+#   This program is distributed in the hope that it will be
+#   useful, but WITHOUT ANY WARRANTY; without even the implied
+#   warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+#   PURPOSE. See the GNU General Public License for more details.
+#
+#   You should have received a copy of the GNU General Public
+#   License along with this program; if not, write to the Free
+#   Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+#   Boston, MA 02110-1301, USA.
+#
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+. /usr/lib/beakerlib/beakerlib.sh
+
+PACKAGE="openscap"
+
+rlJournalStart
+
+    rlPhaseStartSetup
+        rlImport --all || rlDie "Failed to import libraries"
+        rlAssertRpm "$PACKAGE"
+
+        RpmSnapshotCreate
+        CleanupRegister "RpmSnapshotRevert"
+        rlRun "TmpDir=\$(mktemp -d)" 0
+        CleanupRegister "rlRun 'rm -r $TmpDir' 0 'Removing tmp directory'"
+        rlRun "cp fix_hugepages_err.patch $TmpDir"
+        rlRun "pushd $TmpDir"
+        CleanupRegister "rlRun 'popd'"
+        rlServiceStart sendmail
+        CleanupRegister "rlRun 'rlServiceRestore sendmail'"
+
+        if rlIsRHEL 7; then
+            rlLog "Build system is using the latest version of python, therefore"
+            rlLog "python3 packages must not be present on the machine to make"
+            rlLog "sure that the build system configures with python2."
+            rlRun "yum remove -y python3*" 0 "python3 packages must not be installed"
+        elif rlIsRHEL ">=8"; then
+            rlFileBackup "/etc/yum.repos.d/"
+            CleanupRegister "rlFileRestore"
+            # we have to enable buildroot repo
+            rlRun "dnf config-manager --set-enabled rhel-buildroot{,-debuginfo,-source}" 0-255
+            # Some packages needed to build openscap moved from rhel-buildroot to rhel-CRB
+            rlRun "dnf config-manager --set-enabled rhel-CRB{,-debuginfo,-source}" 0-255
+            rlRun "dnf config-manager --set-enabled beaker-CRB{,-debuginfo,-source}" 0-255
+            # AppStream source repo is not enabled by default
+            rlRun "dnf config-manager --set-enabled beaker-AppStream-source" 0-255
+        fi
+
+        rlFetchSrcForInstalled $PACKAGE
+        if rlIsFedora || rlIsRHEL ">=8"; then
+            # --nobest is used as a workaround for the bz#1677583
+            rlRun "dnf --nobest builddep -y $PACKAGE*"
+        else
+            rlRun "yum-builddep -y $PACKAGE*"
+        fi
+        TOPDIR=`rpm --eval %_topdir`
+        rlRun "rm -rf ${TOPDIR}/BUILD/${PACKAGE}*" 0-255
+        rlRun "rpm -ihv `ls *.rpm`" 0 "Install $PACKAGE source RPM"
+    rlPhaseEnd
+
+    rlPhaseStartSetup "Prepare upstream test suite (%prep and %build stages from the spec file)"
+        rlRun "rpmbuild -v -bc ${TOPDIR}/SPECS/${PACKAGE}.spec"
+        CleanupRegister "rlRun 'rm -rf ${TOPDIR}/BUILD/${PACKAGE}*'"
+        CleanupRegister "rlRun 'rm -rf ${TOPDIR}/SPECS/${PACKAGE}*'"
+        CleanupRegister "rlRun 'rm -rf ${TOPDIR}/SOURCES/*'"
+        if rlIsRHEL 7; then
+            rlLogInfo "Waive the known issue with hugepages on ppc64/ppc64le"
+            rlLogInfo "platforms which won't be fixed in RHEL7:"
+            rlLogInfo "https://bugzilla.redhat.com/show_bug.cgi?id=1607382"
+            TEST_CWD=$(pwd)
+            TREE_CWD=$(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*)
+            rlRun "cp fix_hugepages_err.patch $TREE_CWD"
+            rlRun "cd $TREE_CWD"
+            rlRun "patch -p1 < fix_hugepages_err.patch"
+            rlRun "cd $TEST_CWD" 0 "Restore test default working directory"
+        fi
+    rlPhaseEnd
+
+    rlPhaseStartTest "Run upstream test suite against installed $PACKAGE"
+        rlRun "export OSCAP_FULL_VALIDATION=1"
+        rlRun "export CUSTOM_OSCAP=$(which oscap)"
+        if rlIsFedora || rlIsRHEL ">=8"; then
+            rlRun -s "cmake --build $(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*/build) --target test"
+            rv=$?
+            # cmake output does not contain enough information, submit verbose output log
+            FILE="$TOPDIR/BUILD/openscap*/build/Testing/Temporary/LastTest.log"
+            rlFileSubmit $(readlink -f $FILE) test_verbose_output.log
+        else
+            rlRun -s "make -C $(readlink -f ${TOPDIR}/BUILD/${PACKAGE}*) check"
+            rv=$?
+        fi
+
+        # if we got error, submit file with result of particular test for easier debugging
+        if [ $rv -ne 0 ]; then
+            if rlIsFedora || rlIsRHEL ">=8"; then
+                FILE="${TOPDIR}/BUILD/${PACKAGE}*/build/tests"
+                rlBundleLogs test_outputs_all $(readlink -f $FILE)
+            elif grep "See tests/" $rlRun_LOG; then
+                RESULT=`grep "See tests/" $rlRun_LOG | sed -e "s/See tests\(.*\)/tests\1/"`
+                FILE="${TOPDIR}/BUILD/${PACKAGE}*/${RESULT}"
+                cat $(readlink -f $FILE)
+                rlBundleLogs test_outputs $(dirname $FILE)
+                if grep "See tests/" $FILE; then # we need to go deeper ...
+                    RESULT=`grep "See tests/" $FILE | sed -e "s/See tests\(.*\)/tests\1/"`
+                    FILE="${TOPDIR}/BUILD/${PACKAGE}*/${RESULT}"
+                    cat $(readlink -f $FILE)
+                    rlBundleLogs test_outputs_deep $(dirname $FILE)
+                fi
+            fi
+        fi
+    rlPhaseEnd
+
+    rlPhaseStartCleanup
+        CleanupDo
+    rlPhaseEnd
+
+rlJournalPrintText
+rlJournalEnd