diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/ansible-packaging.spec b/ansible-packaging.spec index 9a9e255..08cb792 100644 --- a/ansible-packaging.spec +++ b/ansible-packaging.spec @@ -1,6 +1,8 @@ +%global _docdir_fmt ansible-packaging + Name: ansible-packaging Version: 1 -Release: 8.1%{?dist} +Release: 19.1%{?dist} Summary: RPM packaging macros and generators for Ansible collections License: GPL-3.0-or-later @@ -48,7 +50,7 @@ Requires: /usr/bin/ansible-test Requires: %{py3_dist pytest} Requires: %{py3_dist pytest-mock} Requires: %{py3_dist pytest-xdist} -Requires: %{py3_dist pytest-forked} +Requires: (%{py3_dist pytest-forked} if ansible-core < 2.16~~) Requires: %{py3_dist pyyaml} # mock is included in the list upstream, but is deprecated in Fedora. # Maintainers should work with upstream to add compat code to support @@ -114,23 +116,24 @@ errors rpm_eval -E '%%ansible_collection_url' echo echo echo "Ensure macro works when both arguments are passed and no control macros are set" -[[ $(rpm_eval -E '%%ansible_collection_url community general') == \ - "https://galaxy.ansible.com/community/general" ]] +[ "$(rpm_eval -E '%%ansible_collection_url community general')" = \ + "https://galaxy.ansible.com/ui/repo/published/community/general" ] echo echo "Ensure macro works with the control macros" -[[ $(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \ - -E '%%ansible_collection_url') == "https://galaxy.ansible.com/ansible/posix" ]] +[ "$(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \ + -E '%%ansible_collection_url')" = \ + "https://galaxy.ansible.com/ui/repo/published/ansible/posix" ] echo echo "Ensure macro prefers the collection namespace and name passed as an argument over the control macros" -[[ $(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \ - -E '%%ansible_collection_url community general') == "https://galaxy.ansible.com/community/general" ]] +[ "$(rpm_eval -D 'collection_namespace ansible' -D 'collection_name posix' \ + -E '%%ansible_collection_url community general')" = \ + "https://galaxy.ansible.com/ui/repo/published/community/general" ] %files -%license COPYING %{_fileattrsdir}/ansible.attr %{_rpmmacrodir}/macros.ansible %{_rpmconfigdir}/ansible-generator @@ -138,17 +141,60 @@ echo "Ensure macro prefers the collection namespace and name passed as an argume %files -n ansible-srpm-macros +%license COPYING %{_rpmmacrodir}/macros.ansible-srpm # ansible-core in RHEL 8.6 is built against python38. In c8s and the next RHEL # 8 minor release, it will be built against python39. The testing dependencies # are not yet packaged for either python version in EPEL 8. -%if ! (%{defined rhel} && 0%{?rhel} < 9) +# +# The ansible-test binary is unshipped in EL 10, so we cannot ship the tests +# subpackage yet. +# https://issues.redhat.com/browse/RHEL-69915 +%if %{undefined el8} && %{undefined el10} %files tests %endif %changelog +* Fri Jan 16 2026 Fedora Release Engineering - 1-19.1 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_44_Mass_Rebuild + +* Wed Jul 23 2025 Fedora Release Engineering - 1-18.1 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + +* Thu Jan 16 2025 Fedora Release Engineering - 1-17.1 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild + +* Tue Dec 17 2024 Maxwell G - 1-16.1 +- Temporarily disable tests subpackage on EL 10 + +* Wed Jul 17 2024 Fedora Release Engineering - 1-16 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + +* Sat Feb 10 2024 Maxwell G - 1-15 +- %%ansible_test_unit: handle test dependencies on other collections + +* Mon Jan 22 2024 Fedora Release Engineering - 1-14 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Fri Jan 19 2024 Fedora Release Engineering - 1-13 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Tue Dec 05 2023 Maxwell G - 1-12 +- %%ansible_collection_url: handle new URL scheme + +* Wed Jul 19 2023 Fedora Release Engineering - 1-11 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + +* Thu May 11 2023 Maxwell G - 1-10 +- %%ansible_collection_install - disable spurious collections path warnings +- ansible-packaging-tests - don't depend on pytest-forked with ansible-core 2.16 +- ansible-srpm-macros - include license file in the package + +* Wed Jan 18 2023 Fedora Release Engineering - 1-9.1 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + * Sat Sep 24 2022 Maxwell G - 1-8.1 - Refactor %%ansible_collection_url, %%ansible_collection_install, %%ansible_test_unit. @@ -159,13 +205,6 @@ echo "Ensure macro prefers the collection namespace and name passed as an argume - Prepare to deprecate %%ansible_collection_files - Undefine %%_package_note_file to stop that file from leaking into collection artifacts. -- Require ansible-core at buildtime now that the source of the conflict - has been addressed. - Relates: 2121892 - -* Sat Aug 27 2022 Maxwell G - 1-8 -- Allow both ansible-core and ansible at buildtime again -- Fixes: rhbz#2121892. * Mon Aug 01 2022 Maxwell G - 1-7 - Implement %%ansible_test_unit and add ansible-packaging-tests metapackage. diff --git a/ansible_collection.py b/ansible_collection.py index 70093be..c549283 100755 --- a/ansible_collection.py +++ b/ansible_collection.py @@ -19,12 +19,15 @@ the provided arguments. """ import argparse +import json +import os import shutil import subprocess import sys +from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory -from typing import Any, Dict, Optional, Sequence, Union +from typing import Any, Dict, Optional, Sequence, Tuple, Union from yaml import CSafeLoader, load @@ -63,6 +66,7 @@ class AnsibleCollection: "ansible-galaxy", "collection", "install", + "--force", "-n", "-p", str(destdir), @@ -83,25 +87,98 @@ class AnsibleCollection: with open(filelist, "w", encoding="utf-8") as file: file.write(contents) - def unit_test(self, extra_args: Sequence) -> None: - with TemporaryDirectory() as temp: - temppath = Path(temp) / "ansible_collections" / self.namespace / self.name + def unit_test( + self, + extra_args: Sequence[str], + extra_paths: Sequence[Path], + collections: Sequence[str], + ) -> None: + with TemporaryDirectory() as _temp: + temp = Path(_temp) + temppath = temp / "ansible_collections" / self.namespace / self.name shutil.copytree( self.collection_srcdir, temppath, ) + collection_paths = ( + self._get_collection_path(collection) for collection in collections + ) + for extra in chain(collection_paths, extra_paths): + self._handle_extra_path(temp, extra) args = ("ansible-test", "units", *extra_args) print(f"Running: {args}") print() # Without this, the print statements are shown after the command # output when building in mock. sys.stdout.flush() - subprocess.run(args, cwd=temppath, check=True) + subprocess.run( + args, + cwd=temppath, + check=True, + env={**os.environ, "ANSIBLE_GALAXY_COLLECTIONS_PATH_WARNING": "0"}, + ) + + def _get_collection_path(self, collection: str) -> Path: + proc = subprocess.run( + ["ansible-galaxy", "collection", "list", "--format=json", collection], + check=True, + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + # { + # "/usr/share/ansible/collections/ansible_collections": { + # "community.general": { + # "version": "8.2.0" + # } + # } + # } + data: Dict[str, Dict[str, Any]] = json.loads(proc.stdout) + for path, collection_part in data.items(): + version = collection_part[collection]["version"] + print(f"Using locally-installed version {version} of {collection}") + return Path(path, *collection.split(".", 1)) + raise CollectionError(f"Failed to add {collection} to the test tree") + + def _handle_extra_path(self, collection_tree: Path, extra_path: Path) -> None: + namespace_name = _get_namespace_name(extra_path) + if namespace_name == (self.namespace, self.name): + raise CollectionError( + f"{extra_path} is the same collection as {self.collection_srcdir}" + ) + new_path = Path(collection_tree, "ansible_collections", *namespace_name) + if new_path.is_dir(): + raise CollectionError( + f"Cannot copy {extra_path}." + f" Collection {namespace_name} was already added." + ) + print( + f"Copying {extra_path} ({'.'.join(namespace_name)}) to the collection tree" + ) + shutil.copytree(extra_path, new_path) + + +def _get_namespace_name(extra_path: Path) -> Tuple[str, str]: + data_file = extra_path / "MANIFEST.json" + data_file2 = extra_path / "galaxy.yml" + if data_file.is_file(): + with data_file.open("r", encoding="utf-8") as fp: + data = json.load(fp)["collection_info"] + elif data_file2.is_file(): + data_file = data_file2 + with data_file2.open("r", encoding="utf-8") as fp: + data = load(fp, Loader=CSafeLoader) + else: + raise CollectionError(f"No metadata file found for collection in {extra_path}") + expected_keys = {"namespace", "name"} + if set(data) & expected_keys != expected_keys: + raise CollectionError(f"Invalid metadata file: {data_file}") + return data["namespace"], data["name"] def parseargs() -> argparse.Namespace: parser = argparse.ArgumentParser( - "Install and test Ansible Collections in an rpmbuild environment" + description="Install and test Ansible Collections in an rpmbuild environment" ) subparsers = parser.add_subparsers(dest="action") install_parser = subparsers.add_parser( @@ -126,13 +203,33 @@ def parseargs() -> argparse.Namespace: help="Run ansible-test unit after creating the necessary directory structure", ) test_parser.add_argument( - "extra_args", nargs="*", help="Extra arguments to pass to ansible-test" + "-p", + "--extra-path", + dest="extra_paths", + action="append", + help="Path to an extra collection include in the test ansible_collection tree", + type=Path, ) - args = parser.parse_args() + test_parser.add_argument( + "-c", + "--collection", + action="append", + dest="collections", + help="Add a collection from the collection path to the test tree", + ) + test_parser.set_defaults(allow_extra_args=True) + args, extra_args = parser.parse_known_args() # add_subparsers does not support required on Python 3.6 if not args.action: parser.print_usage() sys.exit(2) + if extra_args: + if not getattr(args, "allow_extra_args", False): + parser.error(f"unrecognized arguments: {' '.join(extra_args)}") + if extra_args and extra_args[0] == "--": + extra_args = extra_args[1:] + args.extra_args = extra_args + vars(args).pop("allow_extra_args", None) return args @@ -143,11 +240,15 @@ def main(): collection.install(args.collections_dir) collection.write_filelist(args.filelist) elif args.action == "test": - collection.unit_test(args.extra_args) + collection.unit_test( + args.extra_args, + (args.extra_paths or ()), + (args.collections or ()), + ) if __name__ == "__main__": try: main() except (CollectionError, subprocess.CalledProcessError) as err: - sys.exit(err) + sys.exit(str(err)) diff --git a/gating.yaml b/gating.yaml new file mode 100644 index 0000000..5480e33 --- /dev/null +++ b/gating.yaml @@ -0,0 +1,10 @@ +--- !Policy +product_versions: + - fedora-* +decision_contexts: + - bodhi_update_push_testing + - bodhi_update_push_stable +subject_type: koji_build +rules: + - !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional} +... diff --git a/macros.ansible b/macros.ansible index db2269a..2cc3c2e 100644 --- a/macros.ansible +++ b/macros.ansible @@ -16,9 +16,9 @@ --filelist %{ansible_collection_filelist} } -%ansible_test_unit() %{shrink: -%{_rpmconfigdir}/ansible_collection.py test -- ---python-interpreter %{__python3} --local %{?*} +%ansible_test_unit(p:c:) %{shrink: +%{_rpmconfigdir}/ansible_collection.py test +%{**} --python-interpreter %{__python3} --local } # TODO: Officially deprecate this macro and add the following line to the macro diff --git a/macros.ansible-srpm b/macros.ansible-srpm index 0adacd7..fdf2ce5 100644 --- a/macros.ansible-srpm +++ b/macros.ansible-srpm @@ -8,6 +8,9 @@ # either or approach. Both arguments must be passed OR both control macros must # be defined. + +%__ansible_galaxy_collection_url https://galaxy.ansible.com/ui/repo/published + %ansible_collection_url() %{lua: local namespace_name = nil if rpm.expand("%collection_namespace") ~= "%collection_namespace" @@ -21,5 +24,6 @@ rpm.expand("%{error:%%ansible_collection_url: You must pass the collection " .. "namespace as the first arg and the collection name as the second}") end - print("https://galaxy.ansible.com/" .. namespace_name) + url = rpm.expand("%__ansible_galaxy_collection_url") + print(url .. "/" .. namespace_name) } diff --git a/tests/rebuild.fmf b/tests/rebuild.fmf new file mode 100644 index 0000000..a1414be --- /dev/null +++ b/tests/rebuild.fmf @@ -0,0 +1,16 @@ +summary: Rebuild dependents to ensure that builds pass +discover: + - name: Rebuild collection packages + how: shell + tests: + - name: Rebuild ansible-collection-community-docker + test: tests/rebuild.sh ansible-collection-community-docker + require: + - ansible-core + - ansible-packaging + - ansible-srpm-macros + - fedpkg + - rpm-build + - sudo +execute: + how: tmt diff --git a/tests/rebuild.sh b/tests/rebuild.sh new file mode 100755 index 0000000..fbbb8b8 --- /dev/null +++ b/tests/rebuild.sh @@ -0,0 +1,27 @@ +#!/usr/bin/bash -x +set -euo pipefail + +package="${1}" + +cat <<'EOF' > ~/.rpmmacros +%_topdir %(echo $HOME)/rpmbuild +%_sourcedir %(pwd) +%_srcrpmdir %(pwd) +%_rpmdir %(pwd)/rpms +EOF + +mkdir -p clones +cd clones +fedpkg clone -a "${package}" +cd "${package}" +fedpkg srpm +sudo dnf builddep -y *.src.rpm +rc=0 +rpmbuild --rebuild *.src.rpm | tee build.log || rc=$? + +# move the results to the artifacts directory, so we can examine them +artifacts="${TEST_ARTIFACTS:-/tmp/artifacts}" +mkdir -p "${artifacts}" +mv -v *.rpm rpms/*/* build.log "${artifacts}/" || : + +exit "${rc}" diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..ee08526 --- /dev/null +++ b/tox.ini @@ -0,0 +1,34 @@ +[tox] +env_list = + formatters + lint + typing-py{36,39,311,312} + +[testenv:formatters] +description = Run formatters +skip_install = true +deps = + isort + black +commands = + black {posargs} ansible_collection.py ansible-generator + isort {posargs} ansible_collection.py ansible-generator + +[testenv:lint] +description = Run linters +skip_install = true +deps = + ruff +commands = + ruff check {posargs} ansible_collection.py ansible-generator + +[testenv:typing-py{36,39,311,312}] +description = Run type checkers +skip_install = true +deps = + mypy + types-pyyaml +commands = + mypy {posargs} ansible_collection.py ansible-generator +set_env = + PYTHONPATH=${PWD}