diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 16613b0..05857b3 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,7 +1,7 @@ #!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then +if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then exit 0 fi @@ -10,7 +10,7 @@ fi path_to_fix=${1:?} # First, check that the parser is available: -if [ ! -x /usr/bin/marshalparser ]; then +if [[ ! -x /usr/bin/marshalparser ]]; then echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" exit 1 fi diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 1f7c4cd..1911fa1 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -6,7 +6,7 @@ errors_terminate=$2 # Therefore $1 ($default_python) is not needed and is invoked with "" by default. # $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 -if [ 0$extra -eq 1 ]; then +if [[ 0"$extra" -eq 1 ]]; then echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr exit 1 fi @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then exit 0 fi @@ -36,7 +36,7 @@ function python_bytecompile() { local options=$1 local python_binary=$2 - local exclude=$3 + # local exclude=$3 # No longer used local python_libdir="$4" local compileall_flags="$5" @@ -45,14 +45,14 @@ function python_bytecompile() # # Python 3.4 and higher # - if [ "$python_version" -ge 34 ]; then + if [[ "$python_version" -ge 34 ]]; then # We compile all opt levels in one go: only when $options is empty. - if [ -n "$options" ]; then + if [[ -n "$options" ]]; then return fi - if [ "$python_version" -ge 39 ]; then + if [[ "$python_version" -ge 39 ]]; then # For Pyhon 3.9+, use the standard library compileall_module=compileall else @@ -60,7 +60,7 @@ function python_bytecompile() compileall_module=compileall2 fi - if [ "$python_version" -ge 37 ]; then + if [[ "$python_version" -ge 37 ]]; then # Force the TIMESTAMP invalidation mode invalidation_option=--invalidation-mode=timestamp else @@ -69,18 +69,14 @@ function python_bytecompile() invalidation_option= fi - [ ! -z $exclude ] && exclude="-x '$exclude'" - # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels # -q disables verbose output # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -96,13 +92,10 @@ python_libdir = "$python_libdir" depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" -exclude = r"$exclude" class Filter: def search(self, path): ret = not os.path.realpath(path).startswith(build_root) - if exclude: - ret = ret or re.search(exclude, path) return ret sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) @@ -137,12 +130,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +143,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo new file mode 100755 index 0000000..b72d704 --- /dev/null +++ b/brp-python-rpm-in-distinfo @@ -0,0 +1,15 @@ +#!/usr/bin/bash + +set -eu +# If using normal root, avoid changing anything. +if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then + exit 0 +fi + +find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do + if cmp -s <(echo pip) "$installer" ; then + echo "rpm" > "$installer" + rm -f "$(dirname "$installer")/RECORD" + fi +done +exit 0 diff --git a/import_all_modules.py b/import_all_modules.py index 3930236..97e924c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,6 +7,7 @@ import os import re import site import sys +import traceback from contextlib import contextmanager from pathlib import Path @@ -93,11 +94,24 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. + + Return a list of failed modules. ''' + failed_modules = [] + for module in modules: print('Check import:', module, file=sys.stderr) - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception: + traceback.print_exc(file=sys.stderr) + failed_modules.append(module) + + if failed_modules: + print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) + + return failed_modules def argparser(): @@ -164,7 +178,10 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - import_modules(modules) + failed_modules = import_modules(modules) + + if failed_modules: + raise SystemExit(1) if __name__ == '__main__': diff --git a/macros.python b/macros.python index 0ba4d78..4bd0da6 100644 --- a/macros.python +++ b/macros.python @@ -23,6 +23,23 @@ end print(_python_macro_cache[cache_key][name]) } +# Deprecation wrapper, warns only once per macro +# Options: +# -n - The name of the macro that is deprecated +%_python_deprecated(n:) %{lua: +if not _python_deprecated_warned then + -- This is intentionally a global lua table + _python_deprecated_warned = {} +end +if not _python_deprecated_warned[opt.n] then + _python_deprecated_warned[opt.n] = true + local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. + "See the current Python packaging guidelines: " .. + "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" + macros.warn({msg}) +end +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) @@ -69,17 +86,17 @@ print(_python_macro_cache[cache_key][name]) # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{expand:\\\ +%py_build() %{_python_deprecated -n py_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_wheel() %{expand:\\\ +%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{expand:\\\ +%py_install() %{_python_deprecated -n py_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python-srpm b/macros.python-srpm index 00098d2..d5cd4e5 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.13 +%__default_python3_version 3.14 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. @@ -66,6 +66,8 @@ ### BRP scripts (and related macros) +## Modifies installation method in .dist-info/INSTALLER file to rpm +%python_rpm_in_distinfo 1 ## Automatically compile python files %py_auto_byte_compile 1 ## Should python bytecompilation errors terminate a build? @@ -78,16 +80,19 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts +%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink ## This macro is included in redhat-rpm-config's %%__os_install_post # Note that the order matters: -# 1. brp-python-bytecompile can create (or replace) pyc files -# 2. brp-fix-pyc-reproducibility can modify the pyc files from above -# 3. brp-python-hardlink de-duplicates identical pyc files +# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file +# 2. brp-python-bytecompile can create (or replace) pyc files +# 3. brp-fix-pyc-reproducibility can modify the pyc files from above +# 4. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ + %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ @@ -225,7 +230,7 @@ end } -%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' @@ -238,6 +243,7 @@ local value_F = rpm.expand('%{-F}') local value_a = rpm.expand('%{-a}') local value_A = rpm.expand('%{-A}') + local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -260,7 +266,8 @@ if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname @@ -280,7 +287,7 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom new file mode 100644 index 0000000..41389f9 --- /dev/null +++ b/macros.python-wheel-sbom @@ -0,0 +1,124 @@ +# The macros in this file are used to add SBOM to wheel files that we ship. +# Majority of Python packages will not need to do that, +# as they only use wheels as an intermediate artifact. +# The macros will be used by packages installing wheel to %%python_wheel_dir +# or by Python interpreters bundling their own (patched) wheels. +# +# The runtime dependencies are not Required by the python-rpm-macros package, +# users of this macro need to specify them on their own or rely on the fact that +# they are all available in the default buildroot. +# +# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS +# +# The wheels are modified in-place. + + +# Path of the SBOM file in the PEP 770 .dist-info/sboms directory +# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ +# section Recognized file patterns +%__python_wheel_sbom_filename bom.json + + +# The SBOM content to put to the file +# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 +%__python_wheel_sbom_content %{expand:{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "%{name}", + "version": "%{version}-%{release}", + "purl": "%{__python_wheel_purl}" + } + ] +}} + + +# The purl used above +# We use the src package name (which is easier to get and more useful to consumers). +# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 +# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ +%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src + + +# The purl namespace used above +# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ +# Distributors, define %%dist_purl_namespace to set this. +# The rest of the code is fallback for distributions without it (relying on %%dist_name). +%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: + if macros.epel then + -- being epel beats the %%dist_name value + -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 + print("epel") + else + local dist_map = { + -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm + -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 + ["Fedora Linux"] = "fedora", + -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 + ["CentOS Stream"] = "centos", + -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ + ["Red Hat Enterprise Linux"] = "redhat", + -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html + ["AlmaLinux"] = "almalinux", + -- from https://github.com/google/osv.dev/pull/2939 + ["Rocky Linux"] = "rocky-linux", + } + print(dist_map[macros.dist_name] or "unknown") + end +}} + + +# A Bash scriptlet to inject the SBOM file into the wheel(s) +# The macro takes positional nargs+ with wheel paths +# For each wheel, it +# 1. aborts if the SBOM file is already there (it won't override) +# 2. inserts the SBOM file to .dist-info/sboms +# 3. amends .dist-info/RECORD with the added SBOM file +%python_wheel_inject_sbom() %{expand:( + %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] + + set -eu -o pipefail + export LANG=C.utf-8 + + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + pwd0=$(pwd) + ret=0 + + for whl in %{*}; do + cd "$tmpdir" + if [[ "$whl" != /* ]]; then + whl="$pwd0/$whl" + fi + + record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') + distinfo="${record%%/RECORD}" + bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" + + if zipinfo -1 "$whl" | grep -qFx "$bom"; then + echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 + ret=1 + continue + fi + + unzip "$whl" "$record" + mkdir "$distinfo/sboms" + echo '%{__python_wheel_sbom_content}' > "$bom" + checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" + size="$(wc --bytes "$bom" | cut -f1 -d' ')" + echo "$bom,$checksum,$size" >> "$record" + + if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then + touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" + fi + + zip -r "$whl" "$record" "$bom" + rm -rf "$distinfo" + cd "$pwd0" + done + + exit $ret +)} + diff --git a/macros.python3 b/macros.python3 index 7197433..38cdcd4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -43,17 +43,17 @@ # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{expand:\\\ +%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_wheel() %{expand:\\\ +%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{expand:\\\ +%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/pathfix.py b/pathfix.py index 1d7db3a..6808382 100644 --- a/pathfix.py +++ b/pathfix.py @@ -56,7 +56,6 @@ def main(): if recursedown(arg): bad = 1 elif os.path.islink(arg): err(arg + ': will not process symbolic links\n') - bad = 1 else: if fix(arg): bad = 1 sys.exit(bad) diff --git a/plan.fmf b/plan.fmf new file mode 100644 index 0000000..88850dd --- /dev/null +++ b/plan.fmf @@ -0,0 +1,40 @@ +execute: + how: tmt + +discover: + - name: same_repo + how: shell + tests: + - name: pytest + test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - name: manual_byte_compilation_clamp_mtime_off + path: /tests + test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - name: manual_byte_compilation_clamp_mtime_on + path: /tests + test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - name: rpmlint_clamp_mtime_off + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: rpmlint_clamp_mtime_on + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: python_wheel_inject_sbom + path: /tests + test: rpmbuild -ba testwheel.spec + +prepare: + - name: Install dependencies + how: install + package: + - rpm-build + - rpmlint + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-setuptools + - python3-pip + - python3-pytest + - python3.6 + - dnf + - name: Update packages + how: shell + script: dnf upgrade -y diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bcae6d1..263853a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -8,6 +8,7 @@ Source101: macros.python Source102: macros.python-srpm Source104: macros.python3 Source105: macros.pybytecompile +Source106: macros.python-wheel-sbom # Lua files Source201: python.lua @@ -33,6 +34,8 @@ Source402: brp-python-hardlink # This one is from redhat-rpm-config < 190 # It has no upstream yet Source403: brp-fix-pyc-reproducibility +# brp script to write "rpm" string into the .dist-info/INSTALLER file +Source404: brp-python-rpm-in-distinfo # macros and lua: MIT # import_all_modules.py: MIT @@ -53,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -136,6 +139,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} %global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} %global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} +%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} %check @@ -146,6 +150,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/pathfix.py @@ -156,6 +161,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -163,6 +169,40 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Oct 16 2025 Miro Hrončok - 3.14-9 +- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content +- That way, accidentally unpackaged files within are reported as errors + +* Tue Sep 09 2025 Miro Hrončok - 3.14-8 +- %%python_extras_subpkg: Add -v option to specify the required version(-release) +- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) + +* Fri Aug 29 2025 Miro Hrončok - 3.14-7 +- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos + +* Wed Aug 13 2025 Miro Hrončok - 3.14-6 +- Introduce %%python_wheel_inject_sbom + +* Mon Aug 11 2025 Lumír Balhar - 3.14-5 +- import_all_modules: Add error handling for import failures + +* Fri Jul 25 2025 Fedora Release Engineering +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + +* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 +- pathfix.py: Don't fail on symbolic links + +* Sun Jun 29 2025 Miro Hrončok - 3.14-2 +- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install +- Deprecate %%py_build, %%py_build_wheel, and %%py_install +- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros + +* Wed May 28 2025 Karolina Surma - 3.14-1 +- Update main Python to 3.14 + +* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 +- Add brp script to modify .dist-info/INSTALLER file + * Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index e9fa44a..e6cc8f7 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -551,7 +551,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -562,7 +562,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected @@ -658,6 +658,21 @@ def test_python_extras_subpkg_aA(): 'BuildArch: noarch (default)) options are not possible') +def test_python_extras_subpkg_v(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 1.2.3 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') @@ -760,6 +775,9 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) + # strip the deprecation message + if 'deprecated' in lines[0]: + lines = lines[1:] assert lines[0] == ( 'error: attempt to use unversioned python, ' 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' @@ -777,7 +795,9 @@ def test_unversioned_python_errors(macro): @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + unverisoned = rpm_eval(macro, __python='/usr/bin/python3') + expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] + assert unverisoned == expected # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -958,3 +978,30 @@ def test_multi_python3(alt_x_y): lines = rpm_eval(evals) lines = [l for l in lines if l] # strip empty lines generated by %global assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +@pytest.mark.parametrize('macro', [ + '%py3_build', + '%py3_build_wheel', + '%py3_install', +]) +def test_deprecation(macro): + lines = rpm_eval(macro) + assert "is deprecated" in lines[0] + assert f"{macro} " in lines[0] + + +def test_multiple_deprecation(): + source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' + lines = rpm_eval(source) + + assert "is deprecated" in lines[0] + assert "%py3_build " in lines[0] + + assert "is deprecated" in lines[1] + assert "%py3_build_wheel " in lines[1] + + assert "is deprecated" in lines[2] + assert "%py3_install " in lines[2] + + assert "is deprecated" not in '\n'.join(lines[3:]) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 8b89bc4..dd10689 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['this_is_a_module_in_cwd']) @@ -175,7 +175,7 @@ def test_modules_from_files_are_found(tmp_path): def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,24 +253,70 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_module_with_error_propagates_exception(tmp_path): +def test_import_module_returns_failed_modules(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + assert failed_modules == ['this_is_a_module_in_tmp_path_2'] + + +def test_module_with_error_propagates_exception(tmp_path, capsys): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - # The correct exception must be raised - with pytest.raises(ZeroDivisionError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) + _, err = capsys.readouterr() + assert "ZeroDivisionError" in err + + +def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) + assert failed_modules == [] + + +def test_all_modules_are_imported(tmp_path, capsys): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + with pytest.raises(SystemExit): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'missing_module', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ]) + _, err = capsys.readouterr() + for i in range(1, 4): + assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err + assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py new file mode 100644 index 0000000..7546dc0 --- /dev/null +++ b/tests/test_rpm_in_distinfo.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import os +import pytest +import subprocess + +@pytest.fixture +def create_test_files(tmp_path): + def _create(subpath, installer_content): + dir_path = tmp_path / subpath + dir_path.mkdir(parents=True, exist_ok=True) + installer_file = dir_path / "INSTALLER" + installer_file.write_text(installer_content) + record_file = dir_path / "RECORD" + record_file.write_text("dummy content in RECORD file\n") + return dir_path + return _create + +testdata = [ + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), +] +@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) +def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, +path, installer_content, expected_installer_content, record_file_exists): + script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") + tmp_dir = create_test_files(path, installer_content) + monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) + result = subprocess.run( + [script_path], + capture_output=True, text=True + ) + + assert result.returncode == 0 + assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content + assert Path(tmp_dir / "RECORD").exists() is record_file_exists + diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index af2c2a1..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - tags: - - classic - tasks: - - dnf: - name: "*" - state: latest - -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - tests: - - pytest: - dir: . - run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation_clamp_mtime_off: - dir: . - run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - - manual_byte_compilation_clamp_mtime_on: - dir: . - run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec - - rpmlint_clamp_mtime_off: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - rpmlint_clamp_mtime_on: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - required_packages: - - rpm-build - - rpmlint - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-pytest - - python3.6 - diff --git a/tests/testwheel.spec b/tests/testwheel.spec new file mode 100644 index 0000000..461d7f3 --- /dev/null +++ b/tests/testwheel.spec @@ -0,0 +1,116 @@ +Name: testwheel +Epoch: 42 +Version: 1 +Release: 0%{?dist} +Summary: ... +License: MIT +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools >= 61 +BuildRequires: python3-pip + +%description +This builds and installs a wheel which we can then use as a test for +%%python_wheel_inject_sbom. + + +%prep +cat > pyproject.toml << EOF +[project] +name = "testwheel" +version = "1" + +[build-system] +requires = ["setuptools >= 61"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["testwheel*"] +EOF +# create a secondary dist-info folder in the project +# we need to ensure this file is not altered +mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info +touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +echo 'recursive-include testwheel/_vendor *' > MANIFEST.in + + +%build +export PIP_CONFIG_FILE=/dev/null +%{python3} -m pip wheel . --no-build-isolation + +# The macro should happily alter multiple wheels, let's make more +for i in {1..5}; do + mkdir ${i} + cp -a *.whl ${i} +done + +# using relative paths should succeed +%python_wheel_inject_sbom {1..5}/*.whl + +# repetitive use should bail out and fail (SBOM is already there) +%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true + +# each wheel should already have it, all should fail individually as well +for i in {1..5}; do + %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true +done + + +%install +mkdir -p %{buildroot}%{python_wheel_dir} +cp -a *.whl %{buildroot}%{python_wheel_dir} + +# using absolute paths should work +%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl + +# and fail when repeated +%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true + + +%check +%define venvsite venv/lib/python%{python3_version}/site-packages +%{python3} -m venv venv +venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl + +test -f %{venvsite}/testwheel-1.dist-info/RECORD +test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD +# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) +sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' + +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true + +# this deliberately uses a different mechanism than the macro +# if you are running this test on a different distro, adjust it +%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} + +PYTHONOPTIMIZE=0 %{python3} -c " +import json +with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: + sbom = json.load(fp) +assert len(sbom['components']) == 1 +assert sbom['components'][0]['type'] == 'library' +assert sbom['components'][0]['name'] == 'testwheel' +assert sbom['components'][0]['version'] == '1-0%{?dist}' +assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' +" + +# replace the installation with the original unaltered wheel +venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl +test -f %{venvsite}/testwheel-1.dist-info/RECORD +# no SBOM +test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true + + +%files +%{python_wheel_dir}/*.whl + + +%changelog +* Wed Aug 13 2025 Miro Hrončok - 42:1-0 +- A static changelog with a date, so we can clamp mtimes