diff --git a/Sanity/Run-internal-BIND-test-suite/Makefile b/Sanity/Run-internal-BIND-test-suite/Makefile index ee5ee21..5c1b35f 100644 --- a/Sanity/Run-internal-BIND-test-suite/Makefile +++ b/Sanity/Run-internal-BIND-test-suite/Makefile @@ -1,8 +1,9 @@ # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # -# Makefile of /CoreOS/bind/Sanity/Run-internal-BIND-test-suite +# Makefile of tests/Run-internal-BIND-test-suite # Description: Run internal BIND test suite # Author: Martin Cermak +# Author: Petr Mensik # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # @@ -54,14 +55,19 @@ $(METADATA): Makefile @echo "Type: Sanity" >> $(METADATA) @echo "TestTime: 8h" >> $(METADATA) @echo "RunFor: bind" >> $(METADATA) - @echo "Requires: bind perl-Net-DNS rpm-build bind-utils" >> $(METADATA) + @echo "Requires: bind rpm-build bind-utils" >> $(METADATA) + @echo "Requires: perl-Net-DNS perl-Net-DNS-Nameserver" >> $(METADATA) + @echo "Requires: perl-Time-HiRes" >> $(METADATA) @echo "Requires: bind-pkcs11 bind-pkcs11-utils softhsm" >> $(METADATA) @echo "Requires: openssl-devel libtool autoconf" >> $(METADATA) - @echo "Requires: libcap-devel libidn-devel libxml2-devel" >> $(METADATA) - @echo "Requires: openldap-devel postgresql-devel" >> $(METADATA) - @echo "Requires: sqlite-devel krb5-devel net-tools perl-Net-DNS-Nameserver" >> $(METADATA) - @echo "Requires: yum-utils dnf-utils" >> $(METADATA) - @echo "Requires: gcc-c++" >> $(METADATA) +# Try to satisfy all build dependencies from here + @echo "Requires: bind-devel" >> $(METADATA) + @echo "Requires: net-tools" >> $(METADATA) + @echo "Requires: dnf-utils" >> $(METADATA) + @echo "Requires: kyua" >> $(METADATA) + @echo "Requires: libcmocka-devel" >> $(METADATA) +# Obsolete, uses cmocka +# @echo "Requires: libatf-c gcc-c++" >> $(METADATA) @echo "Priority: Normal" >> $(METADATA) @echo "License: GPLv2" >> $(METADATA) @echo "Confidential: no" >> $(METADATA) diff --git a/Sanity/Run-internal-BIND-test-suite/PURPOSE b/Sanity/Run-internal-BIND-test-suite/PURPOSE index 3fcf82b..754ba2a 100644 --- a/Sanity/Run-internal-BIND-test-suite/PURPOSE +++ b/Sanity/Run-internal-BIND-test-suite/PURPOSE @@ -1,4 +1,4 @@ -PURPOSE of /CoreOS/bind/Sanity/Run-internal-BIND-test-suite +PURPOSE of tests/Run-internal-BIND-test-suite Description: Run internal BIND test suite Author: Martin Cermak Bug summary: Run internal BIND test suite diff --git a/Sanity/Run-internal-BIND-test-suite/knownerror b/Sanity/Run-internal-BIND-test-suite/knownerror index f9618b6..2d0c8e9 100644 --- a/Sanity/Run-internal-BIND-test-suite/knownerror +++ b/Sanity/Run-internal-BIND-test-suite/knownerror @@ -1 +1,2 @@ A:System test dlz +A:System test idna diff --git a/Sanity/Run-internal-BIND-test-suite/knownerror.el6 b/Sanity/Run-internal-BIND-test-suite/knownerror.el6 deleted file mode 100644 index 1e49e21..0000000 --- a/Sanity/Run-internal-BIND-test-suite/knownerror.el6 +++ /dev/null @@ -1,2 +0,0 @@ -A:System test rrl -A:System test autosign diff --git a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.0 b/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.0 deleted file mode 100644 index b590526..0000000 --- a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.0 +++ /dev/null @@ -1,3 +0,0 @@ -dlz -dnssec -idna diff --git a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.1 b/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.1 deleted file mode 100644 index eefa932..0000000 --- a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.1 +++ /dev/null @@ -1,3 +0,0 @@ -dlz -dnssec -mkeys diff --git a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.2 b/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.2 deleted file mode 100644 index 58401c0..0000000 --- a/Sanity/Run-internal-BIND-test-suite/knownerror.rhel-8.2 +++ /dev/null @@ -1 +0,0 @@ -dlz diff --git a/Sanity/Run-internal-BIND-test-suite/runtest.sh b/Sanity/Run-internal-BIND-test-suite/runtest.sh index 13bd330..7066344 100755 --- a/Sanity/Run-internal-BIND-test-suite/runtest.sh +++ b/Sanity/Run-internal-BIND-test-suite/runtest.sh @@ -2,7 +2,7 @@ # vim: dict=/usr/share/beakerlib/dictionary.vim cpt=.,w,b,u,t,i,k # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # -# runtest.sh of /CoreOS/bind/Sanity/Run-internal-BIND-test-suite +# runtest.sh of tests/Run-internal-BIND-test-suite # Description: Run internal BIND test suite # Author: Martin Cermak # @@ -25,122 +25,100 @@ # Boston, MA 02110-1301, USA. # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Usable variables: -# override make target to start testing on more threads -# MAKE_TEST='-j4 test' -# Do not clean existing build if already built -# REUSE_BUILD=y -# Make retest faster, skip build if possible -# QUICK=y # Include rhts environment . /usr/bin/rhts-environment.sh . /usr/lib/beakerlib/beakerlib.sh -PACKAGE="" -rpm -q bind && PACKAGE="bind" -rpm -q bind97 && PACKAGE="bind97" +PACKAGE="bind" + +# Set those variables to n to skip tests on variants +#DEFAULT_VARIANTS="normal pkcs11 sdb" +DEFAULT_VARIANTS="normal" +#TEST_VARIANTS="normal" + +# +# Runs test suite and checks known errors +# Prepared to be repeated with another variants +run_testsuite() +{ + local RESULT_TEXT="$TMPDIR/test${NAMED_VARIANT}.txt" + local FOUNDERROR=`mktemp found-XXXXXXXX.err` + local KNOWNERROR=/dev/null + + if [ -f "$ORIG/knownerror${NAMED_VARIANT}.$TAG" ]; then + KNOWNERROR=`readlink -f $ORIG/knownerror.$TAG` + elif [ -f "$ORIG/knownerror${NAMED_VARIANT}" ]; then + KNOWNERROR=`readlink -f $ORIG/knownerror` + fi + + # Sometime it can fail. Report just failures that are not known + rlRun "make test -j${CORES:-1} &> $RESULT_TEXT" 0-255 "Perform the test." + rlRun "grep -C 10 FAIL $RESULT_TEXT" 0-255 "Quickly show the test error (if any)." + + rlRun "$FILTER $RESULT_TEXT" 0 "Showing unsuccessful tests" + rlRun "$FILTER -s $RESULT_TEXT > $FOUNDERROR" 0 + rlRun "ls $KNOWNERROR $FOUNDERROR $RESULT_TEXT" 0 'check if there is needed files' + rlLog "`cat $FOUNDERROR`" + + rlAssertLesserOrEqual "Checking number of found errors is in limits" "$(grep '^FAIL' $FOUNDERROR | wc -l)" "$(wc -l <$KNOWNERROR)" + cat $FOUNDERROR | while read STATUS TEST ; do + if [ "$STATUS" = FAIL ]; then + rlRun "grep '$TEST' $KNOWNERROR" 0 "Check $TEST failure is expected" + else + rlLog "$STATUS $TEST" + fi + done +} rlJournalStart rlPhaseStartSetup - rpm -q perl-Net-DNS-Nameserver || yum install -y perl-Net-DNS-Nameserver - rlLog "`rpm -q perl-Net-DNS-Nameserver`" # package assertions rlAssertRpm $PACKAGE rlAssertRpm rpm-build + rlAssertRpm perl-Net-DNS-Nameserver #pwd ORIG=`pwd` - FOUNDERROR=`mktemp` SETUP_SOFTHSM=`readlink -f setup-named-softhsm.sh` FILTER=`readlink -f bind-systest-filter.sh` + CORES=`grep 'processor\s*:' /proc/cpuinfo | wc -l` TAG=generic if [ -f /etc/os-release ]; then # extract platform tag - VERSION_ID=`(source /etc/os-release && echo ${ID}-${VERSION_ID})` TAG=`(source /etc/os-release && echo ${PLATFORM_ID#platform:})` - else - rlIsRHEL '6' && TAG=el6 - fi - - if [ -f "knownerror.$VERSION_ID" ]; then - KNOWNERROR=`readlink -f knownerror.$VERSION_ID` - elif [ -f "knownerror.$TAG" ]; then - KNOWNERROR=`readlink -f knownerror.$TAG` - elif [ -f "knownerror" ]; then - KNOWNERROR=`readlink -f knownerror` - fi - - if [ -n "$QUICK" ]; then - REUSE_BUILD=y - MAKE_TEST='test -j4' fi #tempdir rlRun "TMPDIR=\`mktemp -d\`" 0 "Creating tmp directory" rlRun "pushd $TMPDIR" - if rlIsRHEL 8 && dnf config-manager --help >/dev/null; then - # Some build dependencies are not in repositories enabled - # by default: libidn2-devel, softshm - dnf config-manager --set-enabled rhel-buildroot - #not checking return code: 1mt and beaker uses different names for repo - fi - # topdir - if rlIsRHEL 3 || rlIsRHEL 4 || rlIsRHEL 5; then - TOPDIR="/usr/src/redhat" - else - TOPDIR="/root/rpmbuild" - fi + TOPDIR=`rpm -E '%{_topdir}'` # cleanup in topdir mkdir -p $TOPDIR/{BUILD,SOURCES,SPECS} - if [ "$REUSE_BUILD" != y ] - then - rlRun "rm -rf $TOPDIR/{BUILD,SOURCES,SPECS}/*" - else - rlLog "Not cleaning previous build" - fi + rm -rf $TOPDIR/{BUILD,SOURCES,SPECS}/* # download src rpm - rlFetchSrcForInstalled "$PACKAGE" if ! ls bind*.src.rpm; then - rlRun "yumdownloader --source bind" 0 "Trying alternative fetch from repository" + rlRun "dnf --enablerepo='*-source' download --source bind" 0 "Fetch source from repository" rlRun "rpm -i bind*.src.rpm" fi rlRun "rpm --define '_topdir $TOPDIR' -Uvh *rpm &> $TMPDIR/install.txt" rlRun "cd $TOPDIR/SPECS" - # softhsm is no longer in normal repository. Enable idm module on RHEL8 to make softhsm module available - if rlIsRHEL 8 - then - rlRun "dnf -y module enable idm:DL1" - fi - - if dnf builddep --help >/dev/null; then - rlRun "dnf -y builddep --nobest *.spec" - elif which yum-builddep; then - rlRun "yum-builddep -y *.spec" - else - rlWarn "there is nor yum-utils neither dnf-utils for install dependencies, ENJOY!" - fi + rlRun "dnf -y builddep *.spec" # stop bind if it is running - service named stop + rlServiceStop named rlPhaseEnd rlPhaseStartTest - if [ "$REUSE_BUILD" = y ] && ls -1 "$TOPDIR/BUILD"/bind* > /dev/null - then - rlLog "Skipping bind build" - else - # rebuild from source - rlRun "rpmbuild -ba *.spec &> $TMPDIR/build.txt" 0 "Building bind" - fi + # rebuild from source + rlRun "rpmbuild -ba *.spec &> $TMPDIR/build.txt" # the test rlRun "cd $TOPDIR/BUILD/bind*" @@ -150,7 +128,7 @@ rlJournalStart rlRun "chown -R root ." if [ -x "$SETUP_SOFTHSM" ]; then - rlRun "eval $(bash $SETUP_SOFTHSM -A)" 0 "Preparing PKCS#11 token slot" + rlRun "eval \"$(bash $SETUP_SOFTHSM -A)\"" 0 "Preparing PKCS#11 token slot" rlRun "pkcs11-tokens" 0 "Testing token slot availability" else rlLog "PKCS#11 not initialized" @@ -168,75 +146,40 @@ rlJournalStart export LC_ALL=en_US.UTF-8 rlRun "pushd $BUILD" - # keep separate results on 9.11+ - rlRun "sed -e 's/testsummary.sh/& -n/' -i bin/tests/system/Makefile" 0 "Modify to keep results" - # dlz test is broken because specific build way we use. It is supported only by named-sdb - # but that is not even tested by testsuite - rlRun "sed -e 's/ dlz / /' -i bin/tests/system/Makefile" 0 "Skip always failing dlz test" - # Try to fix tssgsig failures on some machines, do not use system kerberos configuration - export KRB5_CONFIG=/dev/null - # Allow speedup by defining MAKE_TEST='test -j4' on multiprocessor machines - rlRun "make ${MAKE_TEST:-test} &> $TMPDIR/test.txt" 0-255 "Perform the test." - export -n KRB5_CONFIG - - # This would catch just errors on 9.11+ - if [ -f bin/tests/system/testsummary.sh ]; then - FAILED_TESTS=`grep '^R:[a-z0-9_-][a-z0-9_-]*:FAIL' $TMPDIR/test.txt | cut -d':' -f2 | sort | xargs echo` - if [ -n "$FAILED_TESTS" ]; then - rlLog "Failed tests: $FAILED_TESTS" - rlRun "tar czf $TMPDIR/failed-artifacts.tar.gz -C bin/tests/system $FAILED_TESTS" 0 "Archiving failed artifacts in tests" - else - rlLog "No failed tests" - fi - else - FAILED_TESTS=:any: - rlRun "tar czf $TMPDIR/failed-artifacts.tar.gz bin/tests/system" 0 "Archiving all system tests" + if echo "${TEST_VARIANTS:-$DEFAULT_VARIANTS}" | grep -q normal; then + rlLog "Running normal variant" + export NAMED_VARIANT= DNSSEC_VARIANT= + run_testsuite + rlLog "Finished normal variant" fi - rlRun "popd" - rlRun "grep -C 10 FAIL $TMPDIR/test.txt" 0-255 "Quickly show the test error (if any)." + if echo "${TEST_VARIANTS:-$DEFAULT_VARIANTS}" | grep -q sdb; then + rlLog "Running sdb variant" + export NAMED_VARIANT=-sdb DNSSEC_VARIANT= + run_testsuite + rlLog "Finished sdb variant" + fi + + if echo "${TEST_VARIANTS:-$DEFAULT_VARIANTS}" | grep -q pkcs11; then + rlLog "Running pkcs11 variant" + # Unfortunately, PKCS11 variant uses shared key storage + # It cannot use more threads for that reason + export NAMED_VARIANT=-pkcs11 DNSSEC_VARIANT=-pkcs11 + CORES=1 run_testsuite + rlLog "Finished pkcs11 variant" + fi + + rlRun "popd" rlRun "./bin/tests/system/ifconfig.sh down" 0 "Remove fake network interfaces." - - #list of failures: - rlRun "$FILTER $TMPDIR/test.txt" 0 "Showing unsuccessful tests" - rlRun "$FILTER -s $TMPDIR/test.txt > $FOUNDERROR" 0 - rlRun "ls $KNOWNERROR $FOUNDERROR $TMPDIR/test.txt" 0 'check if there are needed files' - rlLog "`echo list;cat $FOUNDERROR`" - - FAILED_FOUND="$(grep '^FAIL' $FOUNDERROR | wc -l)" - FAILED_KNOWN="$(wc -l <$KNOWNERROR)" - rlAssertLesserOrEqual "Checking number of found errors is in limits" "$FAILED_FOUND" "$FAILED_KNOWN" - cat $FOUNDERROR | while read STATUS TEST ; do - if [ "$STATUS" = FAIL ]; then - rlRun "grep '$TEST' $KNOWNERROR" 0 "Check $TEST failure is expected" - else - rlLog "$STATUS $TEST" - fi - done - - if [ "$FAILED_TESTS" = ':any:' ] && [ "$FAILED_FOUND" -le "$FAILED_KNOWN" ] - then - # Newer version produces archive only when some error occured - rlLog "No error found, not uploading artifacts" - rm -f "$TMPDIR/failed-artifacts.tar.gz" - fi rlPhaseEnd - rlPhaseStartCleanup "`echo RESULT_ ;cat $FOUNDERROR|grep 'FAIL'`" - #this phase is only due to report to webUI without needs of open file - rlLog "`echo RESULT_ ;cat $FOUNDERROR|grep FAIL`" - rlPhaseEnd - rlPhaseStartCleanup - rlBundleLogs "TEST_LOGS" "$TMPDIR/install.txt" "$TMPDIR/builddeps.txt" "$TMPDIR/build.txt" "$TMPDIR/test.txt" - if [ -r "$TMPDIR/failed-artifacts.tar.gz" ]; then - rlFileSubmit "$TMPDIR/failed-artifacts.tar.gz" failed-artifacts.tar.gz - fi + rlBundleLogs "BUILD_LOGS" "$TMPDIR/install.txt" "$TMPDIR/builddeps.txt" "$TMPDIR/build.txt" + rlBundleLogs "TEST_LOGS" "$TMPDIR"/test*.txt rlRun "popd" rlRun "rm -r $TMPDIR" 0 "Removing tmp directory" - rlRun "rm -rf $FOUNDERROR" rlPhaseEnd rlJournalEnd