tests: Add llvm dist-git tests

* Enable fmf in the repo
* Copied tests from rpms/llvm
* Integrated llvm-devel-cmake which was not in rpms/llvm

.fmf/version

@@ -0,0 +1 @@
+1

README.md

@@ -0,0 +1,3 @@
+# llvm
+
+Tests for llvm

binutils-plugin-ar/main.fmf

@@ -0,0 +1,20 @@
+summary: Make sure the the LLVM plugin for binutils' ar works
+test: "$WITH_SCL ./test.sh"
+require:
+  - binutils
+adjust:
+  # Common requirements when LLVM is not SCL-ized
+  - require+:
+      - llvm
+      - clang
+    when: collection is not defined
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - llvm-toolset-13.0-llvm
+      - llvm-toolset-13.0-clang
+    when: collection == llvm-toolset-13.0
+  - require+:
+      - llvm-toolset-14.0-llvm
+      - llvm-toolset-14.0-clang
+    when: collection == llvm-toolset-14.0

binutils-plugin-ar/test.sh

@@ -1,4 +1,6 @@
-set -exo pipefail
+#!/bin/sh -eux
+
+set pipefail
 
 echo "void lto_function(){}" | clang -flto -O2 -c -x c -o foo.o -
 ar crs foo.a foo.o

build-compat/main.fmf

@@ -0,0 +1,3 @@
+summary: Make sure rebuilding LLVM still works with the new LLVM
+test: "$WITH_SCL ./test.sh"
+enabled: false

build-compat/test.sh

@@ -0,0 +1,14 @@
+#!/bin/sh -eux
+
+dnf download --disablerepo=* --enablerepo=test-llvm --source llvm
+
+# The src.rpm is available in the directory the test run from.
+set +e
+mock --resultdir=. --old-chroot --with compat_build --rebuild *.src.rpm
+if [ $? -ne 0 ]; then
+  cat root.log
+  cat build.log
+  exit 1
+fi
+
+exit 0

build-gating-cmake.fmf

@@ -0,0 +1,39 @@
+#
+# Build/PR gating tests for LLVM
+#
+# Compatible with various LLVM distributions:
+#
+#   * Fedora (ursine packages)
+#   * Centos 9 stream (ursine packages)
+#   * RHEL-9 (ursine packages)
+#   * RHEL-8 (Red Hat module)
+#   * RHEL-7 (software collection)
+#
+
+summary: LLVM tests for build/PR gating
+adjust:
+  - because: "Plan to be ran when either executed locally, or executed by CI system to gate a build or PR."
+    when: >-
+      trigger is defined
+      and trigger != commit
+      and trigger != build
+    enabled: false
+
+  - because: "When testing SCL-ized LLVM, the collection must be enabled first"
+    environment+:
+      WITH_SCL: "scl enable llvm-toolset-13.0"
+    when: "collection == llvm-toolset-13.0"
+  - because: "When testing SCL-ized LLVM, the collection must be enabled first"
+    environment+:
+      WITH_SCL: "scl enable llvm-toolset-14.0"
+    when: "collection == llvm-toolset-14.0"
+
+discover:
+    - name: llvm-tests
+      how: fmf
+      test: llvm-devel-cmake
+execute:
+    how: tmt
+provision:
+  hardware:
+    memory: ">= 4 GiB"

build-gating.fmf

@@ -0,0 +1,58 @@
+#
+# Build/PR gating tests for LLVM
+#
+# Compatible with various LLVM distributions:
+#
+#   * Fedora (ursine packages)
+#   * Centos 9 stream (ursine packages)
+#   * RHEL-9 (ursine packages)
+#   * RHEL-8 (Red Hat module)
+#   * RHEL-7 (software collection)
+#
+
+summary: LLVM tests for build/PR gating
+adjust:
+  - because: "Plan to be ran when either executed locally, or executed by CI system to gate a build or PR."
+    when: >-
+      trigger is defined
+      and trigger != commit
+      and trigger != build
+    enabled: false
+
+  - because: "When testing SCL-ized LLVM, the collection must be enabled first"
+    environment+:
+      WITH_SCL: "scl enable llvm-toolset-13.0 rust-toolset-1.58"
+    when: "collection == llvm-toolset-13.0"
+  - because: "When testing SCL-ized LLVM, the collection must be enabled first"
+    environment+:
+      WITH_SCL: "scl enable llvm-toolset-14.0 rust-toolset-1.62"
+    when: "collection == llvm-toolset-14.0"
+
+  # Unfortunately, TMT does not support more declarative approach, we need to run commands on our own.
+  - because: "On CentOS, CRB must be enabled to provide rarer packages"
+    prepare+:
+      - name: Enable CRB
+        how: shell
+        script: dnf config-manager --set-enabled crb
+    when: >-
+      distro == centos
+
+  # Unfortunately, TMT does not support more declarative approach, we need to run commands on our own.
+  - because: "On RHEL, CRB must be enabled to provide rarer packages"
+    prepare+:
+      - name: Enable CRB
+        how: shell
+        script: dnf config-manager --set-enabled rhel-CRB
+    when: >-
+      distro == rhel-9
+      or distro == rhel-8
+
+discover:
+    - name: llvm-tests
+      how: fmf
+      filter: "tag:-spoils-installation"
+execute:
+    how: tmt
+provision:
+  hardware:
+    memory: ">= 4 GiB"

integration-test-suite/main.fmf

@@ -0,0 +1,81 @@
+summary: Run the upstream LLVM integration test suite
+description: |
+  Run the upstream LLVM integration test suite
+test: "$WITH_SCL ./test.sh"
+require:
+  - git
+  - make
+  # These require special repositories to be enabled on RHEL
+  - libstdc++-static
+adjust:
+  # Common requirements when LLVM is not SCL-ized
+  - require+:
+      - cmake
+      - clang
+      - clang-analyzer
+      - clang-devel
+      - clang-tools-extra
+      - compiler-rt
+      - lld
+      - lldb
+      - libomp-devel
+      - llvm-devel
+      - python3-lit
+    when: "collection is not defined"
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - llvm-toolset-13.0-cmake
+      - llvm-toolset-13.0-clang
+      - llvm-toolset-13.0-clang-analyzer
+      - llvm-toolset-13.0-clang-devel
+      - llvm-toolset-13.0-clang-tools-extra
+      - llvm-toolset-13.0-compiler-rt
+      - llvm-toolset-13.0-lld
+      - llvm-toolset-13.0-lldb
+      - llvm-toolset-13.0-libomp-devel
+      - llvm-toolset-13.0-llvm-devel
+      - llvm-toolset-13.0-python3-lit
+    when: "collection == llvm-toolset-13.0"
+  - require+:
+      - llvm-toolset-14.0-cmake
+      - llvm-toolset-14.0-clang
+      - llvm-toolset-14.0-clang-analyzer
+      - llvm-toolset-14.0-clang-devel
+      - llvm-toolset-14.0-clang-tools-extra
+      - llvm-toolset-14.0-compiler-rt
+      - llvm-toolset-14.0-lld
+      - llvm-toolset-14.0-lldb
+      - llvm-toolset-14.0-libomp-devel
+      - llvm-toolset-14.0-llvm-devel
+      - llvm-toolset-14.0-python3-lit
+    when: "collection == llvm-toolset-14.0"
+
+  # lld not supported in s390x or ppc64. If any lld package was added to
+  # requirements, remove it.
+  - require-:
+      - lld
+      - llvm-toolset-13.0-lld
+      - llvm-toolset-14.0-lld
+    when: arch == s390x or arch == ppc64
+
+  # libomp not supported in s390x. If any libomp-devel package was added to
+  # requirements, remove it.
+  - require-:
+      - libomp-devel
+      - llvm-toolset-13.0-libomp-devel
+      - llvm-toolset-14.0-libomp-devel
+    when: arch == s390x
+
+  # libcxx shall be required in Fedora, it's not shipped with RHEL.
+  - require+:
+      - libcxx-devel
+    when: "distro == fedora"
+    because: testing against libcxx package in Fedora
+
+  - environment+:
+      CMAKE_CXXLIB: "-DENABLE_LIBCXX=OFF"
+    when: >-
+      distro == centos
+      or distro == rhel
+    because: libcxx is not shipped with neither Centos nor RHEL

integration-test-suite/test.sh

@@ -0,0 +1,39 @@
+#!/bin/sh -eux
+
+usage() {
+    echo "usage `basename $0` [OPTIONS]"
+    echo "  --threads NUM       The number of threads to use for running tests."
+}
+
+threads=0
+thread_args=""
+repo_url=https://github.com/opencollab/llvm-toolchain-integration-test-suite
+
+while [ $# -gt 0 ]; do
+    case $1 in
+        --threads)
+            shift
+            threads="$1"
+            ;;
+        * )
+            echo "unknown option: $1"
+            echo ""
+            usage
+            exit 1
+            ;;
+    esac
+    shift
+done
+
+if [ "$threads" -ge 1 ]; then
+  thread_args="-j$threads"
+fi
+
+cd $(mktemp -d -p /var/tmp)
+git clone $repo_url
+cd llvm-toolchain-integration-test-suite
+mkdir _build && cd _build
+cmake .. ${CMAKE_CXXLIB:-}
+# Do not run with make as it will return 2 in case of failures, which will be
+# reported by tmt as error instead failure. Run with lit so it returns 1.
+lit --show-unsupported --show-xfail -v tests

libllvm-size/main.fmf

@@ -0,0 +1,21 @@
+summary: Check that libLLVM.so has been properly stripped
+test: "$WITH_SCL ./test.sh"
+environment:
+  LIBLLVM_PATH: /usr/lib64/libLLVM.so
+adjust:
+  - require:
+      - llvm-devel
+    when: collection is not defined
+
+  - because: "With SCL-ized LLVM, library path is different"
+    require:
+      - llvm-toolset-13.0-llvm-devel
+    environment+:
+      LIBLLVM_PATH: /opt/rh/llvm-toolset-13.0/root/usr/lib64/libLLVM.so
+    when: "collection == llvm-toolset-13.0"
+  - because: "With SCL-ized LLVM, library path is different"
+    require:
+      - llvm-toolset-14.0-llvm-devel
+    environment+:
+      LIBLLVM_PATH: /opt/rh/llvm-toolset-14.0/root/usr/lib64/libLLVM.so
+    when: "collection == llvm-toolset-14.0"

libllvm-size/test.sh

@@ -0,0 +1,10 @@
+#!/bin/sh -eux
+
+# There is a bug in the build process when it runs out of disk space
+# while stripping binaries, which causes the strip to fail, but does
+# not fail the build.  This results in a libLLVM.so that is over 2GB
+# which breaks the nightly compose.  So this test checks that libLLVM.so
+# is less than 150MB to ensure it was successfully stripped.
+# https://bugzilla.redhat.com/show_bug.cgi?id=1793250
+
+test $(stat -L -c %s $LIBLLVM_PATH) -lt 157286400

llvm-config/main.fmf

@@ -0,0 +1,15 @@
+summary: Make sure llvm-config symlink is properly set up
+test: "$WITH_SCL ./test.sh"
+
+adjust:
+  - require+:
+      - llvm-devel
+    when: collection is not defined
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - llvm-toolset-13.0-llvm-devel
+    when: collection == llvm-toolset-13.0
+  - require+:
+      - llvm-toolset-14.0-llvm-devel
+    when: collection == llvm-toolset-14.0

llvm-config/test.sh

@@ -0,0 +1,3 @@
+#!/bin/sh -eux
+
+llvm-config --version

llvm-devel-cmake/main.fmf

@@ -0,0 +1,35 @@
+summary: Test cmake files from llvm-devel
+test: "$WITH_SCL ./test.sh"
+require:
+  - gcc-c++
+environment:
+  PKG_MGR: "dnf"
+  PKG_QUERY: "dnf repoquery"
+# This test breaks the installation that might have been done in by the plan
+tag:
+  - spoils-installation
+
+adjust:
+  - require+:
+      - cmake
+    when: collection is not defined
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - llvm-toolset-13.0-cmake
+    environment+:
+      COLLECTION_PREFIX: "llvm-toolset-13.0-"
+    when: collection == llvm-toolset-13.0
+  - require+:
+      - llvm-toolset-14.0-cmake
+    environment+:
+      COLLECTION_PREFIX: "llvm-toolset-14.0-"
+    when: collection == llvm-toolset-14.0
+
+  - because: "RHEL 7 and older don't have dnf, fallback to yum"
+    require:
+      - yum-utils
+    environment+:
+      PKG_MGR: "yum"
+      PKG_QUERY: "repoquery"
+    when: distro < rhel-8

llvm-devel-cmake/runtest.sh

@@ -1,16 +0,0 @@
-set -ex
-
-# This test is meant to ensure that the cmake files in llvm-devel work
-# when only the packages it depends on are installed.
-
-ARCH=`rpm --eval '%_arch'`
-
-llvm_devel_num_deps=`dnf repoquery --nvr --requires --resolve llvm-devel.$ARCH | grep '^llvm' | wc -l`
-
-llvm_num_sub_packages_installed=`dnf list installed | grep  '^llvm' | wc -l`
-
-# Verify that only llvm-devel dependencies are installed.
-test `expr $llvm_devel_num_deps + 1` -eq $llvm_num_sub_packages_installed
-
-# Verify that cmake files can me used without errors.
-cmake -G Ninja .

llvm-devel-cmake/test.sh

@@ -0,0 +1,21 @@
+set -ex
+
+# This test is meant to ensure that the cmake files in llvm-devel work
+# when only the packages it depends on are installed.
+# Hence we make sure to remove any llvm-* package and then we install again
+# to check if it works with the dependencies it pulls
+${PKG_MGR} remove -y ${COLLECTION_PREFIX}llvm\*
+
+${PKG_MGR} install -y ${COLLECTION_PREFIX}llvm-devel
+
+ARCH=`rpm --eval '%_arch'`
+
+llvm_devel_num_deps=`${PKG_QUERY} --nvr --requires --resolve ${COLLECTION_PREFIX}llvm-devel.$ARCH | grep "^${COLLECTION_PREFIX}llvm" | wc -l`
+
+llvm_num_sub_packages_installed=`${PKG_MGR} list installed | grep "^${COLLECTION_PREFIX}llvm" | wc -l`
+
+# Verify that only llvm-devel dependencies are installed.
+test `expr $llvm_devel_num_deps + 1` -eq $llvm_num_sub_packages_installed
+
+# Verify that cmake files can me used without errors.
+cmake .

rust-sanity/main.fmf

@@ -0,0 +1,30 @@
+summary: Test basic cargo usage
+test: "$WITH_SCL ./test.sh"
+require: []
+adjust:
+  # Common requirements when LLVM is not SCL-ized
+  - require+:
+    # Required for rust and possibly other dependant packages
+    #
+    # NOTE #1: drop the requirement once the transitional period between releases is gone
+    # and llvm-compat is no longer a) needed, and b) part of the build.
+    #
+    # NOTE #2: keep it the first requirement. Depending on the order of requirements, placing it somewhere
+    # else in the list may lead to harness uninstalling and re-installing LLVM packages. That a) would be
+    # a waste of time, b) could potentially hide issues, leading to testing unexpected NVRs.
+    #
+    #  - llvm-compat
+
+      - cargo
+      - rust
+    when: "collection is not defined"
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - rust-toolset-1.58-cargo
+      - rust-toolset-1.58-rust
+    when: "collection == llvm-toolset-13.0"
+  - require+:
+      - rust-toolset-1.62-cargo
+      - rust-toolset-1.62-rust
+    when: "collection == llvm-toolset-14.0"

rust-sanity/test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eux
+
+cargo new hello
+cd hello
+cargo run

tests-llvm-devel.yml

@@ -1,30 +0,0 @@
-- hosts: localhost
-  pre_tasks:
-  # Make sure we uninstall all sub-packages, so we can test that the cmake files
-  # in llvm-devel are useable when only llvm-devel is installed.
-  - name: Uninstall llvm sub-packages
-    package:
-      name: "{{ item }}"
-      state: absent
-    tags:
-      - classic
-    with_items:
-      - llvm
-      - llvm-devel
-      - llvm-doc
-      - llvm-googletest
-      - llvm-libs
-      - llvm-static
-      - llvm-test
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    required_packages:
-      - llvm-devel
-      - cmake
-      - ninja-build
-      - gcc
-      - gcc-c++
-    tests:
-      - llvm-devel-cmake

tests-llvm-libs.yml

@@ -1,12 +0,0 @@
-- hosts: localhost
-  roles:
-  - role: standard-test-basic
-    tags:
-      - classic
-    required_packages:
-      - clang
-      - binutils
-      - llvm
-      - llvm-libs
-    tests:
-      - binutils-plugin-ar

versioned-llvm-config/main.fmf

@@ -0,0 +1,15 @@
+summary: Make sure the versioned llvm-config symlink is properly set up
+test: "$WITH_SCL ./test.sh"
+
+adjust:
+  - require+:
+      - llvm-devel
+    when: collection is not defined
+
+  # Requirements for SCL-ized LLVM
+  - require+:
+      - llvm-toolset-13.0-llvm-devel
+    when: collection == llvm-toolset-13.0
+  - require+:
+      - llvm-toolset-14.0-llvm-devel
+    when: collection == llvm-toolset-14.0

versioned-llvm-config/test.sh

@@ -0,0 +1,5 @@
+#!/bin/sh -eux
+
+major=$(llvm-config --version | cut -d '.' -f1)
+
+llvm-config-$major --version