diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/README.md b/README.md new file mode 100644 index 0000000..5129148 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# llvm + +Tests for llvm \ No newline at end of file diff --git a/README.rst b/README.rst deleted file mode 100644 index e69de29..0000000 diff --git a/binutils-plugin-ar/main.fmf b/binutils-plugin-ar/main.fmf new file mode 100644 index 0000000..f2363ab --- /dev/null +++ b/binutils-plugin-ar/main.fmf @@ -0,0 +1,20 @@ +summary: Make sure the the LLVM plugin for binutils' ar works +test: "$WITH_SCL ./test.sh" +require: + - binutils +adjust: + # Common requirements when LLVM is not SCL-ized + - require+: + - llvm + - clang + when: collection is not defined + + # Requirements for SCL-ized LLVM + - require+: + - llvm-toolset-13.0-llvm + - llvm-toolset-13.0-clang + when: collection == llvm-toolset-13.0 + - require+: + - llvm-toolset-14.0-llvm + - llvm-toolset-14.0-clang + when: collection == llvm-toolset-14.0 diff --git a/binutils-plugin-ar/runtest.sh b/binutils-plugin-ar/test.sh similarity index 80% rename from binutils-plugin-ar/runtest.sh rename to binutils-plugin-ar/test.sh index d18ab53..5dbacec 100755 --- a/binutils-plugin-ar/runtest.sh +++ b/binutils-plugin-ar/test.sh @@ -1,4 +1,6 @@ -set -exo pipefail +#!/bin/sh -eux + +set pipefail echo "void lto_function(){}" | clang -flto -O2 -c -x c -o foo.o - ar crs foo.a foo.o diff --git a/build-compat/main.fmf b/build-compat/main.fmf new file mode 100644 index 0000000..7e0a3d3 --- /dev/null +++ b/build-compat/main.fmf @@ -0,0 +1,3 @@ +summary: Make sure rebuilding LLVM still works with the new LLVM +test: "$WITH_SCL ./test.sh" +enabled: false diff --git a/build-compat/test.sh b/build-compat/test.sh new file mode 100755 index 0000000..04a97b2 --- /dev/null +++ b/build-compat/test.sh @@ -0,0 +1,14 @@ +#!/bin/sh -eux + +dnf download --disablerepo=* --enablerepo=test-llvm --source llvm + +# The src.rpm is available in the directory the test run from. +set +e +mock --resultdir=. --old-chroot --with compat_build --rebuild *.src.rpm +if [ $? -ne 0 ]; then + cat root.log + cat build.log + exit 1 +fi + +exit 0 diff --git a/build-gating-cmake.fmf b/build-gating-cmake.fmf new file mode 100644 index 0000000..8124048 --- /dev/null +++ b/build-gating-cmake.fmf @@ -0,0 +1,39 @@ +# +# Build/PR gating tests for LLVM +# +# Compatible with various LLVM distributions: +# +# * Fedora (ursine packages) +# * Centos 9 stream (ursine packages) +# * RHEL-9 (ursine packages) +# * RHEL-8 (Red Hat module) +# * RHEL-7 (software collection) +# + +summary: LLVM tests for build/PR gating +adjust: + - because: "Plan to be ran when either executed locally, or executed by CI system to gate a build or PR." + when: >- + trigger is defined + and trigger != commit + and trigger != build + enabled: false + + - because: "When testing SCL-ized LLVM, the collection must be enabled first" + environment+: + WITH_SCL: "scl enable llvm-toolset-13.0" + when: "collection == llvm-toolset-13.0" + - because: "When testing SCL-ized LLVM, the collection must be enabled first" + environment+: + WITH_SCL: "scl enable llvm-toolset-14.0" + when: "collection == llvm-toolset-14.0" + +discover: + - name: llvm-tests + how: fmf + test: llvm-devel-cmake +execute: + how: tmt +provision: + hardware: + memory: ">= 4 GiB" diff --git a/build-gating.fmf b/build-gating.fmf new file mode 100644 index 0000000..c2c56b0 --- /dev/null +++ b/build-gating.fmf @@ -0,0 +1,58 @@ +# +# Build/PR gating tests for LLVM +# +# Compatible with various LLVM distributions: +# +# * Fedora (ursine packages) +# * Centos 9 stream (ursine packages) +# * RHEL-9 (ursine packages) +# * RHEL-8 (Red Hat module) +# * RHEL-7 (software collection) +# + +summary: LLVM tests for build/PR gating +adjust: + - because: "Plan to be ran when either executed locally, or executed by CI system to gate a build or PR." + when: >- + trigger is defined + and trigger != commit + and trigger != build + enabled: false + + - because: "When testing SCL-ized LLVM, the collection must be enabled first" + environment+: + WITH_SCL: "scl enable llvm-toolset-13.0 rust-toolset-1.58" + when: "collection == llvm-toolset-13.0" + - because: "When testing SCL-ized LLVM, the collection must be enabled first" + environment+: + WITH_SCL: "scl enable llvm-toolset-14.0 rust-toolset-1.62" + when: "collection == llvm-toolset-14.0" + + # Unfortunately, TMT does not support more declarative approach, we need to run commands on our own. + - because: "On CentOS, CRB must be enabled to provide rarer packages" + prepare+: + - name: Enable CRB + how: shell + script: dnf config-manager --set-enabled crb + when: >- + distro == centos + + # Unfortunately, TMT does not support more declarative approach, we need to run commands on our own. + - because: "On RHEL, CRB must be enabled to provide rarer packages" + prepare+: + - name: Enable CRB + how: shell + script: dnf config-manager --set-enabled rhel-CRB + when: >- + distro == rhel-9 + or distro == rhel-8 + +discover: + - name: llvm-tests + how: fmf + filter: "tag:-spoils-installation" +execute: + how: tmt +provision: + hardware: + memory: ">= 4 GiB" diff --git a/integration-test-suite/main.fmf b/integration-test-suite/main.fmf new file mode 100644 index 0000000..60504b4 --- /dev/null +++ b/integration-test-suite/main.fmf @@ -0,0 +1,81 @@ +summary: Run the upstream LLVM integration test suite +description: | + Run the upstream LLVM integration test suite +test: "$WITH_SCL ./test.sh" +require: + - git + - make + # These require special repositories to be enabled on RHEL + - libstdc++-static +adjust: + # Common requirements when LLVM is not SCL-ized + - require+: + - cmake + - clang + - clang-analyzer + - clang-devel + - clang-tools-extra + - compiler-rt + - lld + - lldb + - libomp-devel + - llvm-devel + - python3-lit + when: "collection is not defined" + + # Requirements for SCL-ized LLVM + - require+: + - llvm-toolset-13.0-cmake + - llvm-toolset-13.0-clang + - llvm-toolset-13.0-clang-analyzer + - llvm-toolset-13.0-clang-devel + - llvm-toolset-13.0-clang-tools-extra + - llvm-toolset-13.0-compiler-rt + - llvm-toolset-13.0-lld + - llvm-toolset-13.0-lldb + - llvm-toolset-13.0-libomp-devel + - llvm-toolset-13.0-llvm-devel + - llvm-toolset-13.0-python3-lit + when: "collection == llvm-toolset-13.0" + - require+: + - llvm-toolset-14.0-cmake + - llvm-toolset-14.0-clang + - llvm-toolset-14.0-clang-analyzer + - llvm-toolset-14.0-clang-devel + - llvm-toolset-14.0-clang-tools-extra + - llvm-toolset-14.0-compiler-rt + - llvm-toolset-14.0-lld + - llvm-toolset-14.0-lldb + - llvm-toolset-14.0-libomp-devel + - llvm-toolset-14.0-llvm-devel + - llvm-toolset-14.0-python3-lit + when: "collection == llvm-toolset-14.0" + + # lld not supported in s390x or ppc64. If any lld package was added to + # requirements, remove it. + - require-: + - lld + - llvm-toolset-13.0-lld + - llvm-toolset-14.0-lld + when: arch == s390x or arch == ppc64 + + # libomp not supported in s390x. If any libomp-devel package was added to + # requirements, remove it. + - require-: + - libomp-devel + - llvm-toolset-13.0-libomp-devel + - llvm-toolset-14.0-libomp-devel + when: arch == s390x + + # libcxx shall be required in Fedora, it's not shipped with RHEL. + - require+: + - libcxx-devel + when: "distro == fedora" + because: testing against libcxx package in Fedora + + - environment+: + CMAKE_CXXLIB: "-DENABLE_LIBCXX=OFF" + when: >- + distro == centos + or distro == rhel + because: libcxx is not shipped with neither Centos nor RHEL diff --git a/integration-test-suite/test.sh b/integration-test-suite/test.sh new file mode 100755 index 0000000..e350c57 --- /dev/null +++ b/integration-test-suite/test.sh @@ -0,0 +1,39 @@ +#!/bin/sh -eux + +usage() { + echo "usage `basename $0` [OPTIONS]" + echo " --threads NUM The number of threads to use for running tests." +} + +threads=0 +thread_args="" +repo_url=https://github.com/opencollab/llvm-toolchain-integration-test-suite + +while [ $# -gt 0 ]; do + case $1 in + --threads) + shift + threads="$1" + ;; + * ) + echo "unknown option: $1" + echo "" + usage + exit 1 + ;; + esac + shift +done + +if [ "$threads" -ge 1 ]; then + thread_args="-j$threads" +fi + +cd $(mktemp -d -p /var/tmp) +git clone $repo_url +cd llvm-toolchain-integration-test-suite +mkdir _build && cd _build +cmake .. ${CMAKE_CXXLIB:-} +# Do not run with make as it will return 2 in case of failures, which will be +# reported by tmt as error instead failure. Run with lit so it returns 1. +lit --show-unsupported --show-xfail -v tests diff --git a/libllvm-size/main.fmf b/libllvm-size/main.fmf new file mode 100644 index 0000000..e508bfc --- /dev/null +++ b/libllvm-size/main.fmf @@ -0,0 +1,21 @@ +summary: Check that libLLVM.so has been properly stripped +test: "$WITH_SCL ./test.sh" +environment: + LIBLLVM_PATH: /usr/lib64/libLLVM.so +adjust: + - require: + - llvm-devel + when: collection is not defined + + - because: "With SCL-ized LLVM, library path is different" + require: + - llvm-toolset-13.0-llvm-devel + environment+: + LIBLLVM_PATH: /opt/rh/llvm-toolset-13.0/root/usr/lib64/libLLVM.so + when: "collection == llvm-toolset-13.0" + - because: "With SCL-ized LLVM, library path is different" + require: + - llvm-toolset-14.0-llvm-devel + environment+: + LIBLLVM_PATH: /opt/rh/llvm-toolset-14.0/root/usr/lib64/libLLVM.so + when: "collection == llvm-toolset-14.0" diff --git a/libllvm-size/test.sh b/libllvm-size/test.sh new file mode 100755 index 0000000..a9a2dfe --- /dev/null +++ b/libllvm-size/test.sh @@ -0,0 +1,10 @@ +#!/bin/sh -eux + +# There is a bug in the build process when it runs out of disk space +# while stripping binaries, which causes the strip to fail, but does +# not fail the build. This results in a libLLVM.so that is over 2GB +# which breaks the nightly compose. So this test checks that libLLVM.so +# is less than 150MB to ensure it was successfully stripped. +# https://bugzilla.redhat.com/show_bug.cgi?id=1793250 + +test $(stat -L -c %s $LIBLLVM_PATH) -lt 157286400 diff --git a/llvm-config/main.fmf b/llvm-config/main.fmf new file mode 100644 index 0000000..6f8863c --- /dev/null +++ b/llvm-config/main.fmf @@ -0,0 +1,15 @@ +summary: Make sure llvm-config symlink is properly set up +test: "$WITH_SCL ./test.sh" + +adjust: + - require+: + - llvm-devel + when: collection is not defined + + # Requirements for SCL-ized LLVM + - require+: + - llvm-toolset-13.0-llvm-devel + when: collection == llvm-toolset-13.0 + - require+: + - llvm-toolset-14.0-llvm-devel + when: collection == llvm-toolset-14.0 diff --git a/llvm-config/test.sh b/llvm-config/test.sh new file mode 100755 index 0000000..09a0286 --- /dev/null +++ b/llvm-config/test.sh @@ -0,0 +1,3 @@ +#!/bin/sh -eux + +llvm-config --version diff --git a/llvm-devel-cmake/main.fmf b/llvm-devel-cmake/main.fmf new file mode 100644 index 0000000..ec2d017 --- /dev/null +++ b/llvm-devel-cmake/main.fmf @@ -0,0 +1,35 @@ +summary: Test cmake files from llvm-devel +test: "$WITH_SCL ./test.sh" +require: + - gcc-c++ +environment: + PKG_MGR: "dnf" + PKG_QUERY: "dnf repoquery" +# This test breaks the installation that might have been done in by the plan +tag: + - spoils-installation + +adjust: + - require+: + - cmake + when: collection is not defined + + # Requirements for SCL-ized LLVM + - require+: + - llvm-toolset-13.0-cmake + environment+: + COLLECTION_PREFIX: "llvm-toolset-13.0-" + when: collection == llvm-toolset-13.0 + - require+: + - llvm-toolset-14.0-cmake + environment+: + COLLECTION_PREFIX: "llvm-toolset-14.0-" + when: collection == llvm-toolset-14.0 + + - because: "RHEL 7 and older don't have dnf, fallback to yum" + require: + - yum-utils + environment+: + PKG_MGR: "yum" + PKG_QUERY: "repoquery" + when: distro < rhel-8 diff --git a/llvm-devel-cmake/runtest.sh b/llvm-devel-cmake/runtest.sh deleted file mode 100755 index 42d42d3..0000000 --- a/llvm-devel-cmake/runtest.sh +++ /dev/null @@ -1,16 +0,0 @@ -set -ex - -# This test is meant to ensure that the cmake files in llvm-devel work -# when only the packages it depends on are installed. - -ARCH=`rpm --eval '%_arch'` - -llvm_devel_num_deps=`dnf repoquery --nvr --requires --resolve llvm-devel.$ARCH | grep '^llvm' | wc -l` - -llvm_num_sub_packages_installed=`dnf list installed | grep '^llvm' | wc -l` - -# Verify that only llvm-devel dependencies are installed. -test `expr $llvm_devel_num_deps + 1` -eq $llvm_num_sub_packages_installed - -# Verify that cmake files can me used without errors. -cmake -G Ninja . diff --git a/llvm-devel-cmake/test.sh b/llvm-devel-cmake/test.sh new file mode 100755 index 0000000..0d7795f --- /dev/null +++ b/llvm-devel-cmake/test.sh @@ -0,0 +1,21 @@ +set -ex + +# This test is meant to ensure that the cmake files in llvm-devel work +# when only the packages it depends on are installed. +# Hence we make sure to remove any llvm-* package and then we install again +# to check if it works with the dependencies it pulls +${PKG_MGR} remove -y ${COLLECTION_PREFIX}llvm\* + +${PKG_MGR} install -y ${COLLECTION_PREFIX}llvm-devel + +ARCH=`rpm --eval '%_arch'` + +llvm_devel_num_deps=`${PKG_QUERY} --nvr --requires --resolve ${COLLECTION_PREFIX}llvm-devel.$ARCH | grep "^${COLLECTION_PREFIX}llvm" | wc -l` + +llvm_num_sub_packages_installed=`${PKG_MGR} list installed | grep "^${COLLECTION_PREFIX}llvm" | wc -l` + +# Verify that only llvm-devel dependencies are installed. +test `expr $llvm_devel_num_deps + 1` -eq $llvm_num_sub_packages_installed + +# Verify that cmake files can me used without errors. +cmake . diff --git a/rust-sanity/main.fmf b/rust-sanity/main.fmf new file mode 100644 index 0000000..4e050d7 --- /dev/null +++ b/rust-sanity/main.fmf @@ -0,0 +1,30 @@ +summary: Test basic cargo usage +test: "$WITH_SCL ./test.sh" +require: [] +adjust: + # Common requirements when LLVM is not SCL-ized + - require+: + # Required for rust and possibly other dependant packages + # + # NOTE #1: drop the requirement once the transitional period between releases is gone + # and llvm-compat is no longer a) needed, and b) part of the build. + # + # NOTE #2: keep it the first requirement. Depending on the order of requirements, placing it somewhere + # else in the list may lead to harness uninstalling and re-installing LLVM packages. That a) would be + # a waste of time, b) could potentially hide issues, leading to testing unexpected NVRs. + # + # - llvm-compat + + - cargo + - rust + when: "collection is not defined" + + # Requirements for SCL-ized LLVM + - require+: + - rust-toolset-1.58-cargo + - rust-toolset-1.58-rust + when: "collection == llvm-toolset-13.0" + - require+: + - rust-toolset-1.62-cargo + - rust-toolset-1.62-rust + when: "collection == llvm-toolset-14.0" diff --git a/rust-sanity/test.sh b/rust-sanity/test.sh new file mode 100755 index 0000000..fa15399 --- /dev/null +++ b/rust-sanity/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh -eux + +cargo new hello +cd hello +cargo run diff --git a/tests-llvm-devel.yml b/tests-llvm-devel.yml deleted file mode 100644 index 2d5b9af..0000000 --- a/tests-llvm-devel.yml +++ /dev/null @@ -1,30 +0,0 @@ -- hosts: localhost - pre_tasks: - # Make sure we uninstall all sub-packages, so we can test that the cmake files - # in llvm-devel are useable when only llvm-devel is installed. - - name: Uninstall llvm sub-packages - package: - name: "{{ item }}" - state: absent - tags: - - classic - with_items: - - llvm - - llvm-devel - - llvm-doc - - llvm-googletest - - llvm-libs - - llvm-static - - llvm-test - roles: - - role: standard-test-basic - tags: - - classic - required_packages: - - llvm-devel - - cmake - - ninja-build - - gcc - - gcc-c++ - tests: - - llvm-devel-cmake diff --git a/tests-llvm-libs.yml b/tests-llvm-libs.yml deleted file mode 100644 index 1f06af8..0000000 --- a/tests-llvm-libs.yml +++ /dev/null @@ -1,12 +0,0 @@ -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - required_packages: - - clang - - binutils - - llvm - - llvm-libs - tests: - - binutils-plugin-ar diff --git a/versioned-llvm-config/main.fmf b/versioned-llvm-config/main.fmf new file mode 100644 index 0000000..1957100 --- /dev/null +++ b/versioned-llvm-config/main.fmf @@ -0,0 +1,15 @@ +summary: Make sure the versioned llvm-config symlink is properly set up +test: "$WITH_SCL ./test.sh" + +adjust: + - require+: + - llvm-devel + when: collection is not defined + + # Requirements for SCL-ized LLVM + - require+: + - llvm-toolset-13.0-llvm-devel + when: collection == llvm-toolset-13.0 + - require+: + - llvm-toolset-14.0-llvm-devel + when: collection == llvm-toolset-14.0 diff --git a/versioned-llvm-config/test.sh b/versioned-llvm-config/test.sh new file mode 100755 index 0000000..cfe72cf --- /dev/null +++ b/versioned-llvm-config/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh -eux + +major=$(llvm-config --version | cut -d '.' -f1) + +llvm-config-$major --version