From 0411e212490d7cff0feb0565d13177a75b0c85fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Milo=C5=A1=20Prchl=C3=ADk?= Date: Thu, 12 Oct 2023 12:45:53 +0200 Subject: [PATCH] Polish comments, add template and extend README --- README.md | 85 +++++++++++++++++-- .../main.fmf | 2 + .../main.fmf | 2 + .../bugs/RELRO-protection-effective/main.fmf | 9 +- .../main.fmf | 2 + .../main.fmf | 2 +- .../main.fmf | 2 + .../main.fmf | 7 +- .../main.fmf | 15 ++-- .../main.fmf | 1 + .../main.fmf | 7 +- .../main.fmf | 1 + .../main.fmf | 5 +- .../main.fmf | 6 +- .../ld-should-allow-lea-foo-GOT-ecx/main.fmf | 3 +- .../main.fmf | 2 + tests/bugs/ppc64-segv-in-libbfd/main.fmf | 3 + .../main.fmf | 9 +- tests/dts-probe-binaries/main.fmf | 8 +- tests/main.fmf | 56 ++++++++++-- tests/rebuilds/coreutils/main.fmf | 7 +- tests/rebuilds/emacs/main.fmf | 14 +-- tests/rebuilds/kernel/main.fmf | 9 +- tests/rebuilds/wget/main.fmf | 10 +-- tests/selinux-context/main.fmf | 6 +- tests/supported-targets/main.fmf | 6 +- tests/testsuite/main.fmf | 8 +- 27 files changed, 212 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 3e58d24..2818ced 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,86 @@ ## - $ tmt lint --failed-only --outcome-only=fail --outcome-only=warn +```shell +$ tmt lint --failed-only --outcome-only=fail --outcome-only=warn - $ tmt plan show -vv - $ tmt test show -vv +$ tmt plan show -vv +$ tmt test show -vv - $ tmt -c distro=$DISTRO -c arch=$ARCH -vv run discover - $ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/build-gating/common' - $ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/build-gating/kernel-rebuild' - $ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/regression' +$ tmt -c distro=$DISTRO -c arch=$ARCH -vv run discover +$ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/build-gating/common' +$ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/build-gating/kernel-rebuild' +$ tmt -c distro=$DISTRO -c arch=$ARCH -vv run plan -n '/plans/regression' +``` + + +## Adding new tests + +Please, follow these rules: + +* always define `summary` and`duration` fields +* you may define custom `description` field +* always use `+` form of list-like fields (`require`, `link`, ...): + ```yaml + # No: + tag: foo + + tag: + - foo + + # Yes + tag+: + - foo + ``` +* add `verifies` links via `link+` when the test is meant to verify bugzilla or Jira issues +* test applicable to collections only must add `collection-only` tag +* tests sutiable for build gating must add `gate-build` tag + +### Test template + +```yaml +# NOTE: DO NOT LEAVE EMPTY FIELDS, NOT EVEN COMMENTED OUT - REMOVE THEM + +# TODO: Add proper summary +summary: ... + +# TODO: Add a longer description if there's anythign to write about. If not, +# drop the whole field. +# description: ... + +# TODO: Add proper duratio +duration: ... + +# tag+: + # TODO: if this is a test for build gating, add this tag + # - gate-build + + # TODO: if this is a test applicable to collections-only, add this tag + # - collection-only + +# TODO: List bugs verified by the test if there are any. +# link+: +# - verifies: https://bugzilla.redhat.com/... +# - verifies: https://issues.redhat.com/... + +# TODO: Add additional authors - or the only authors if the default contact +# is not applicable, use `contact` in that case. +# contact+: +# - Foo Bar + +# TODO: Add extra required packages +# require+: +# - ... + +# adjust+: + # TODO: If the test is relevant for one architecture only, use the following rule + # - when: arch == ... + # enabled: false + + # TODO: If the test is relevant for collections only, use the following rule + # - when: collection is not defined + # enabled: false +``` ## State of support diff --git a/tests/bugs/FJ7-4-Bug-REG-The-results-of-gprof-command/main.fmf b/tests/bugs/FJ7-4-Bug-REG-The-results-of-gprof-command/main.fmf index b71316c..91ba7dd 100644 --- a/tests/bugs/FJ7-4-Bug-REG-The-results-of-gprof-command/main.fmf +++ b/tests/bugs/FJ7-4-Bug-REG-The-results-of-gprof-command/main.fmf @@ -1,6 +1,8 @@ summary: "[FJ7.4 Bug] [REG] The results of gprof command" + description: | [FJ7.4 Bug]: [REG] The results of gprof command with some options differ between RHEL7.4 beta and RHEL7.3. + duration: 5m link+: diff --git a/tests/bugs/LLNL-7-5-FEAT-RFE-create-an-option-to/main.fmf b/tests/bugs/LLNL-7-5-FEAT-RFE-create-an-option-to/main.fmf index a8e2161..a7464c6 100644 --- a/tests/bugs/LLNL-7-5-FEAT-RFE-create-an-option-to/main.fmf +++ b/tests/bugs/LLNL-7-5-FEAT-RFE-create-an-option-to/main.fmf @@ -1,6 +1,8 @@ summary: "[LLNL 7.5 FEAT] RFE create an option to" + description: | [LLNL 7.5 FEAT] RFE create an option to permanently link in audit library into an executable (glibc) + duration: 15m link+: diff --git a/tests/bugs/RELRO-protection-effective/main.fmf b/tests/bugs/RELRO-protection-effective/main.fmf index 1803d1e..50160d0 100644 --- a/tests/bugs/RELRO-protection-effective/main.fmf +++ b/tests/bugs/RELRO-protection-effective/main.fmf @@ -1,4 +1,5 @@ summary: Exercise PIE and RELRO provided + description: | Test to exercise PIE and RELRO provided by Roland McGrath . @@ -23,11 +24,11 @@ description: | Test without RELRO should pass: $ gcc -pie -fPIE -g -Wl,-z,norelro -o no-relro relro.c $ ./no-relro -contact: - - Milos Prchlik - - Martin Cermak + duration: 15m -# Configurable fields +contact+: + - Martin Cermak + link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1174826 diff --git a/tests/bugs/Rust-TLS-accesses-are-badly-initialized/main.fmf b/tests/bugs/Rust-TLS-accesses-are-badly-initialized/main.fmf index 465bb48..5409693 100644 --- a/tests/bugs/Rust-TLS-accesses-are-badly-initialized/main.fmf +++ b/tests/bugs/Rust-TLS-accesses-are-badly-initialized/main.fmf @@ -1,6 +1,8 @@ summary: Rust TLS accesses are badly initialized + description: | Rust TLS accesses are badly initialized + duration: 15m link+: diff --git a/tests/bugs/aarch64-binutils-relocation-truncated-to-fit/main.fmf b/tests/bugs/aarch64-binutils-relocation-truncated-to-fit/main.fmf index 453f4b4..5b7ca35 100644 --- a/tests/bugs/aarch64-binutils-relocation-truncated-to-fit/main.fmf +++ b/tests/bugs/aarch64-binutils-relocation-truncated-to-fit/main.fmf @@ -1,6 +1,6 @@ summary: "relocation truncated to fit: R_AARCH64_CALL26 (veneers not inserted)" + duration: 15m -# Configurable fields link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1243559 diff --git a/tests/bugs/ar-SEGFAULT-when-creating-static-library-with-lto/main.fmf b/tests/bugs/ar-SEGFAULT-when-creating-static-library-with-lto/main.fmf index b5a02bc..67713fd 100644 --- a/tests/bugs/ar-SEGFAULT-when-creating-static-library-with-lto/main.fmf +++ b/tests/bugs/ar-SEGFAULT-when-creating-static-library-with-lto/main.fmf @@ -1,4 +1,5 @@ summary: ar SEGFAULT when creating static library with lto + duration: 15m link+: @@ -6,4 +7,5 @@ link+: adjust+: - when: arch != x86_64 + because: Test is relevant for x86_64 only enabled: false diff --git a/tests/bugs/binutils-gold-and-binutils-devel-rpms-provide-the-wrong-tags/main.fmf b/tests/bugs/binutils-gold-and-binutils-devel-rpms-provide-the-wrong-tags/main.fmf index fae1a71..18f888c 100644 --- a/tests/bugs/binutils-gold-and-binutils-devel-rpms-provide-the-wrong-tags/main.fmf +++ b/tests/bugs/binutils-gold-and-binutils-devel-rpms-provide-the-wrong-tags/main.fmf @@ -1,17 +1,18 @@ summary: The gcc-toolset binutils-gold RPM provides unprefixed names. + description: | The gcc-toolset-*-binutils-gold RPM provides 'binutils-gold' instead of 'gcc-toolset-*-binutils-gold'. Likewise for the gcc-toolset-*-binutils-devel rpm. -tag: + +tag+: - collection-only + duration: 15m -# Configurable fields link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=2216456 - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=2103056 -# Adjust rules adjust+: - when: collection is not defined enabled: false diff --git a/tests/bugs/binutils-ld-removes-some-R-X86-64-JUMP-SLOT/main.fmf b/tests/bugs/binutils-ld-removes-some-R-X86-64-JUMP-SLOT/main.fmf index 7d1c980..fc4b7dc 100644 --- a/tests/bugs/binutils-ld-removes-some-R-X86-64-JUMP-SLOT/main.fmf +++ b/tests/bugs/binutils-ld-removes-some-R-X86-64-JUMP-SLOT/main.fmf @@ -1,14 +1,15 @@ summary: binutils ld removes some R_X86_64_JUMP_SLOT + description: | ld removes some R_X86_64_JUMP_SLOT relocations -contact: - - Milos Prchlik - - Edjunior Machado -duration: 5m -# Configurable fields -recommend: - - sed +duration: 5m link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1624776 + +contact+: + - Edjunior Machado + +require+: + - sed diff --git a/tests/bugs/binutils-ld-silently-produces-broken-PIE-binaries/main.fmf b/tests/bugs/binutils-ld-silently-produces-broken-PIE-binaries/main.fmf index c2d8f88..636f8f9 100644 --- a/tests/bugs/binutils-ld-silently-produces-broken-PIE-binaries/main.fmf +++ b/tests/bugs/binutils-ld-silently-produces-broken-PIE-binaries/main.fmf @@ -1,4 +1,5 @@ summary: binutils ld silently produces broken PIE binaries + duration: 15m link+: diff --git a/tests/bugs/binutils-support-secondary-relocation-sections/main.fmf b/tests/bugs/binutils-support-secondary-relocation-sections/main.fmf index d706d72..fe53349 100644 --- a/tests/bugs/binutils-support-secondary-relocation-sections/main.fmf +++ b/tests/bugs/binutils-support-secondary-relocation-sections/main.fmf @@ -1,12 +1,13 @@ summary: binutils support secondary relocation sections + description: | support secondary relocation sections -contact: - - Milos Prchlik + +contact+: - Edjunior Machado + duration: 10m -# Configurable fields require+: - grep - gawk diff --git a/tests/bugs/bytecode-stream-in-file-libbfd.a-generated-with-LTO-version-11.3-instead-of-the-expected-13.0/main.fmf b/tests/bugs/bytecode-stream-in-file-libbfd.a-generated-with-LTO-version-11.3-instead-of-the-expected-13.0/main.fmf index cc81f5e..2cb9941 100644 --- a/tests/bugs/bytecode-stream-in-file-libbfd.a-generated-with-LTO-version-11.3-instead-of-the-expected-13.0/main.fmf +++ b/tests/bugs/bytecode-stream-in-file-libbfd.a-generated-with-LTO-version-11.3-instead-of-the-expected-13.0/main.fmf @@ -1,4 +1,5 @@ summary: bytecode stream in file ‘/opt/rh/gcc-toolset-13/root/usr/lib64/libbfd.a’ generated with LTO version 11.3 instead of the expected 13.0 + duration: 10m link+: diff --git a/tests/bugs/dwz-applied-to-a-dts-compiled-binary-complains/main.fmf b/tests/bugs/dwz-applied-to-a-dts-compiled-binary-complains/main.fmf index 9bf58fd..b7f2d92 100644 --- a/tests/bugs/dwz-applied-to-a-dts-compiled-binary-complains/main.fmf +++ b/tests/bugs/dwz-applied-to-a-dts-compiled-binary-complains/main.fmf @@ -1,7 +1,8 @@ summary: dwz applied to a dts-compiled binary complains -contact: - - Milos Prchlik + +contact+: - Sergey Kolosov + duration: 2h link+: diff --git a/tests/bugs/fpie-pie-does-not-appear-to-work-with-__thread-variables/main.fmf b/tests/bugs/fpie-pie-does-not-appear-to-work-with-__thread-variables/main.fmf index f39121e..f5a82c1 100644 --- a/tests/bugs/fpie-pie-does-not-appear-to-work-with-__thread-variables/main.fmf +++ b/tests/bugs/fpie-pie-does-not-appear-to-work-with-__thread-variables/main.fmf @@ -1,9 +1,9 @@ summary: Tests -pie + -fpie with __thread -contact: - - Milos Prchlik + +contact+: - Michal Nowak + duration: 5m -# Configurable fields link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=755872 diff --git a/tests/bugs/ld-should-allow-lea-foo-GOT-ecx/main.fmf b/tests/bugs/ld-should-allow-lea-foo-GOT-ecx/main.fmf index 2b6f97f..3e30dcd 100644 --- a/tests/bugs/ld-should-allow-lea-foo-GOT-ecx/main.fmf +++ b/tests/bugs/ld-should-allow-lea-foo-GOT-ecx/main.fmf @@ -1,8 +1,9 @@ summary: "ld should allow 'lea foo@GOT, %ecx'" + description: | ld should allow "lea foo@GOT, %ecx" + duration: 15m -# Configurable fields link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1573872 diff --git a/tests/bugs/objdump-S-disassembly-code-doesn-t-follow/main.fmf b/tests/bugs/objdump-S-disassembly-code-doesn-t-follow/main.fmf index 3a6e10a..7463d77 100644 --- a/tests/bugs/objdump-S-disassembly-code-doesn-t-follow/main.fmf +++ b/tests/bugs/objdump-S-disassembly-code-doesn-t-follow/main.fmf @@ -1,9 +1,11 @@ summary: objdump -S disassembly code doesn't follow + require+: - strace - coreutils - glibc-debuginfo - coreutils-debuginfo + duration: 15m link+: diff --git a/tests/bugs/ppc64-segv-in-libbfd/main.fmf b/tests/bugs/ppc64-segv-in-libbfd/main.fmf index 152b351..ffcf467 100644 --- a/tests/bugs/ppc64-segv-in-libbfd/main.fmf +++ b/tests/bugs/ppc64-segv-in-libbfd/main.fmf @@ -1,10 +1,13 @@ summary: ppc64 segv in libbfd + description: | ppc64: segv in libbfd + require+: - elfutils - kernel-debuginfo - xz + duration: 30m link+: diff --git a/tests/bugs/rhel8-gold-does-not-resolve-the-address-of-main/main.fmf b/tests/bugs/rhel8-gold-does-not-resolve-the-address-of-main/main.fmf index 0d40870..cf17102 100644 --- a/tests/bugs/rhel8-gold-does-not-resolve-the-address-of-main/main.fmf +++ b/tests/bugs/rhel8-gold-does-not-resolve-the-address-of-main/main.fmf @@ -1,11 +1,12 @@ summary: RHEL8 gold does not resolve the address of main() + description: | rhel8 gold does not resolve the address of main() when it is in a shared library in aarch64 -contact: - - Milos Prchlik - - Edjunior Machado + duration: 5m -# Configurable fields link+: - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=1693661 + +contact+: + - Edjunior Machado diff --git a/tests/dts-probe-binaries/main.fmf b/tests/dts-probe-binaries/main.fmf index 30ef53d..53bc93d 100644 --- a/tests/dts-probe-binaries/main.fmf +++ b/tests/dts-probe-binaries/main.fmf @@ -1,12 +1,13 @@ summary: Toolset binutils on system/toolset built binaries. -tag: - - collection-only + duration: 15m -# Configurable fields # Running outside of SCL context *on purpose* test: "./test.sh" +tag+: + - collection-only + require+: # Requesting *system* versions of packages usually provided by the collection - binutils @@ -23,7 +24,6 @@ require+: environment+: REQUIRES: "binutils gcc gcc-c++ glibc gdb" -# Adjust rules adjust+: - when: collection is not defined enabled: false diff --git a/tests/main.fmf b/tests/main.fmf index 8fb4034..97ef72b 100644 --- a/tests/main.fmf +++ b/tests/main.fmf @@ -1,14 +1,35 @@ +# +# Default values for all tests +# + +# Reset description - not necessarily needed, but it makes the field set to something. description: "" -contact: - - Milos Prchlik -framework: beakerlib - -test: "$WITH_SCL ./test.sh" +# Set component to `binutils` - probably not having any exceptions. component: binutils +# Default set of contacts - some tests will add more as needed to reflect the fact +# some tests were developed by other people. +contact: + - Milos Prchlik + +# No default links, but initialize the field so tests can only *add* links, never set +# the whole field later. link: [] +# No default tags, but initialize the field so tests can only *add* tags, never set +# the whole field later. +tag: [] + +# All tests are using beakerlib +framework: beakerlib + +# All tests are expected to cover collections as well as system binutils. +# Individual tests may override this and avoid collection environment, +# or get disabled via `adjust`. +test: "$WITH_SCL ./test.sh" + +# By default, our internal library is required for all tests. require+: - type: file pattern: @@ -16,16 +37,39 @@ require+: # Adjust rules adjust+: - - when: distro == fedora-rawhide,rhel-9 + - because: "/tmp is not big enough to hold tmpdir for tests" + when: distro == fedora-rawhide,rhel-9 environment+: BU_TMPDIR_PARENT: "/" + # + # Default set of packages under the test, requirements, and related environment variables. + # + # Packages under the test: + # + # * always include corresponding variant of `binutils`, `binutils-devel` and `binutils-gold`. + # + # Requires: + # + # * we start with `gcc`, `gcc-c++`, `glibc` and corresponding `annobin`. + # + # Environments: + # + # * `PACKAGES` shall be only `binutils` or its collection variant and nothing else + # * `_REQUIRES` lists the default requirements - `gcc`, `gcc-c++` and so on + # * `BINUTILS_PACKAGE`, `GCC_PACKAGE` list packages providing active binutils and gcc + # in the test environment. `BINUTILS_PACKAGE` is usually the same as `PACKAGES` but + # but complements `GCC_PACKAGE` + # * `COLLECTION_PACKAGES` lists other binutils packages in the collection + # + # RHEL-9 / no collection - when: distro == rhel-9 and collection is not defined require+: - binutils - binutils-devel - binutils-gold + - glibc - gcc - gcc-c++ - annobin diff --git a/tests/rebuilds/coreutils/main.fmf b/tests/rebuilds/coreutils/main.fmf index 642c0b7..671cf42 100644 --- a/tests/rebuilds/coreutils/main.fmf +++ b/tests/rebuilds/coreutils/main.fmf @@ -1,8 +1,9 @@ summary: Rebuild coreutils -tag: - - gate-build + duration: 1h -# Configurable fields +tag+: + - gate-build + require+: - coreutils diff --git a/tests/rebuilds/emacs/main.fmf b/tests/rebuilds/emacs/main.fmf index 9eeb873..a579fca 100644 --- a/tests/rebuilds/emacs/main.fmf +++ b/tests/rebuilds/emacs/main.fmf @@ -1,8 +1,5 @@ -contact: - - Milos Prchlik - - Martin Cermak - summary: Rebuild Emacs + description: | From: Jeff Law @@ -21,11 +18,14 @@ description: | are necessary since the build process for emacs exercises the problematic bits. -tag: - - gate-build duration: 3h -# Configurable fields +contact+: + - Martin Cermak + +tag+: + - gate-build + require+: - emacs diff --git a/tests/rebuilds/kernel/main.fmf b/tests/rebuilds/kernel/main.fmf index ef9fc3f..7f590c4 100644 --- a/tests/rebuilds/kernel/main.fmf +++ b/tests/rebuilds/kernel/main.fmf @@ -1,4 +1,5 @@ summary: Check whether newly built kernel boots correctly + description: | Rebuild kernel, install it, reboot, and check if we're running the correct kernel. Tailored specificaly for binutils buildroot testing process. @@ -17,11 +18,12 @@ description: | Based on gcc/Sanity/rebuild-kernel by: Author: Michal Nowak Author: Marek Polacek -tag: - - gate-build + duration: 20h -# Configurable fields +tag+: + - gate-build + require+: - yum-utils - rng-tools @@ -45,7 +47,6 @@ require+: - asciidoc - net-tools -# Adjust rules adjust+: - because: "kernel package is gone from Fedora Rawhide default installation" when: distro == fedora-rawhide diff --git a/tests/rebuilds/wget/main.fmf b/tests/rebuilds/wget/main.fmf index 9baf7dc..8ee8938 100644 --- a/tests/rebuilds/wget/main.fmf +++ b/tests/rebuilds/wget/main.fmf @@ -1,11 +1,9 @@ summary: Rebuild wget -tag: - - gate-build + duration: 1h -# Configurable fields +tag+: + - gate-build + require+: - wget - -# Adjust rules -adjust+: [] diff --git a/tests/selinux-context/main.fmf b/tests/selinux-context/main.fmf index 7d3fa5a..1286d8e 100644 --- a/tests/selinux-context/main.fmf +++ b/tests/selinux-context/main.fmf @@ -1,14 +1,14 @@ summary: Verify selinux context of binutils files -require+: - - policycoreutils - link+: - verifies: https://issues.redhat.com/browse/RHELPLAN-166055 - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=2232410 - verifies: https://issues.redhat.com/browse/RHELPLAN-167156 - verifies: https://bugzilla.redhat.com/show_bug.cgi?id=2236729 +require+: + - policycoreutils + adjust+: - when: collection is not defined enabled: false diff --git a/tests/supported-targets/main.fmf b/tests/supported-targets/main.fmf index e89c440..7d82fbb 100644 --- a/tests/supported-targets/main.fmf +++ b/tests/supported-targets/main.fmf @@ -1,6 +1,6 @@ summary: Tests that binutils target support matrix does not regress. -contact: - - Milos Prchlik - - Arjun Shankar duration: 10m + +contact+: + - Arjun Shankar diff --git a/tests/testsuite/main.fmf b/tests/testsuite/main.fmf index a6098b1..378f0b5 100644 --- a/tests/testsuite/main.fmf +++ b/tests/testsuite/main.fmf @@ -1,9 +1,10 @@ summary: Run binutils upstream test suite -tag: - - gate-build + duration: 3h -# Configurable fields +tag+: + - gate-build + require+: - texinfo - glibc-static @@ -16,7 +17,6 @@ require+: - libstdc++-static - rpm-build -# Adjust rules adjust+: - because: "On RHEL-9.3.0 and newer, system binutils build extra binaries for cross compilation" when: distro == rhel-9.3 and collection is not defined