From 2acd7cb9b37be66ac1fe1d85134698d66b6ae170 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 21 Mar 2024 17:02:42 +0100 Subject: [PATCH 01/22] Update bundled compileall2 to version 0.8.0 --- compileall2.py | 80 +++++++++++++++++++++++------------------- python-rpm-macros.spec | 7 ++-- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/compileall2.py b/compileall2.py index c58e545..ea7e76f 100644 --- a/compileall2.py +++ b/compileall2.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, if compiles all modules on sys.path, without +Without arguments, it compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -36,11 +36,11 @@ PY35 = sys.version_info[0:2] >= (3, 5) # introduced in Python 3.7. These cases are covered by variables here or by PY37 # variable itself. if PY37: - pyc_struct_format = '<4sll' + pyc_struct_format = '<4sLL' pyc_header_lenght = 12 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) else: - pyc_struct_format = '<4sl' + pyc_struct_format = '<4sL' pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) @@ -106,7 +106,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path @@ -120,23 +120,34 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, stripdir = dir prependdir = ddir ddir = None - if workers is not None: - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - elif workers != 1: - try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: - workers = 1 + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits + try: + _check_system_limits() + except NotImplementedError: + workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True - if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + if workers != 1 and ProcessPoolExecutor is not None: + mp_context_arg = {} + if PY37: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + mp_context_arg = {"mp_context": mp_context} + # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + **mp_context_arg) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -178,7 +189,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. @@ -190,10 +201,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) - else: - fullname = str(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None @@ -206,13 +215,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: @@ -258,7 +267,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack(*(pyc_header_format + (mtime,))) + expect = struct.pack(*(pyc_header_format + (mtime & 0xFFFF_FFFF,))) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(pyc_header_lenght) @@ -301,9 +310,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) + encoding = sys.stdout.encoding or sys.getdefaultencoding() + msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False @@ -408,8 +416,8 @@ def main(): type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses optimization level of ' - 'Python interpreter itself (specified by -O).')) + 'Default is -1 which uses the optimization level ' + 'of the Python interpreter itself (see -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', @@ -456,7 +464,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: @@ -464,9 +473,6 @@ def main(): print("Error reading file list {}".format(args.flist)) return False - if args.workers is not None: - args.workers = args.workers or None - if PY37 and args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 06fc622..a3c2511 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -13,7 +13,7 @@ Source105: macros.pybytecompile Source201: python.lua # Python code -%global compileall2_version 0.7.1 +%global compileall2_version 0.8.0 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py %global pathfix_version 1.0.0 @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Mar 22 2024 Lumír Balhar - 3.12-8 +- Update bundled compileall2 to version 0.8.0 + * Thu Jan 25 2024 Miro Hrončok - 3.12-7 - %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set From d211646a5e0cc32f62efe963d7970b896445f9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 1 Mar 2024 00:27:31 +0100 Subject: [PATCH 02/22] brp-fix-pyc-reproducibility: use more strict shell style 1. Error out on unset variables. 2. Implement suggestion made by shellcheck: use "||" instead of "-o"" in test. Shellcheck also suggested -print0 to avoid ambiguity with filenames with embedded newlines, but instead use '-exec {} +', which doesn't need xargs but still passes multiple arguments to a single marshalparser invocation. (Those issues are unlikely to cause problems in the rpm environment, but it's nice to be shellcheck-clean to use shellcheck during development.) Co-Authored-By: Benjamin A. Beasley --- brp-fix-pyc-reproducibility | 10 ++++------ python-rpm-macros.spec | 5 ++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 536a126..16613b0 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,13 +1,13 @@ -#!/bin/bash -e +#!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then exit 0 fi # Defined as %py_reproducible_pyc_path macro and passed here as # the first command-line argument -path_to_fix=$1 +path_to_fix=${1:?} # First, check that the parser is available: if [ ! -x /usr/bin/marshalparser ]; then @@ -15,6 +15,4 @@ if [ ! -x /usr/bin/marshalparser ]; then exit 1 fi -# Set pipefail so if $path_to_fix does not exist, the build fails -set -o pipefail -find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite +find "$path_to_fix" -type f -name '*.pyc' -exec /usr/bin/marshalparser --fix --overwrite '{}' '+' diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3c2511..5d2daf2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 +- Minor improvements to brp-fix-pyc-reproducibility + * Fri Mar 22 2024 Lumír Balhar - 3.12-8 - Update bundled compileall2 to version 0.8.0 From a389cc467425766a0122be2c143a9a4fdb76191e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 16 Oct 2023 16:45:02 +0200 Subject: [PATCH 03/22] Update main Python to 3.13 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 278571f..d57e43f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.12 +%__default_python3_version 3.13 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5d2daf2..ffd4d9a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Jun 06 2024 Karolina Surma - 3.13-1 +- Update main Python to 3.13 + * Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 - Minor improvements to brp-fix-pyc-reproducibility From d430201e60a2d911f58bb979b1eca6299bbc96c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 25 Jun 2024 12:34:11 +0200 Subject: [PATCH 04/22] CI tests fix for Python 3.13 We should not unimport pathlib when we use pathlib. This was always fragile. Choose a module we don't use: The failure: ______________________ test_modules_from_files_are_found _______________________ tmp_path = PosixPath('/tmp/pytest-of-root/pytest-0/test_modules_from_files_are_fo0') def test_modules_from_files_are_found(tmp_path): test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') test_file_2.write_text('csv\npathlib\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): sys.modules.pop(m, None) > modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) test_import_all_modules.py:170: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/rpm/redhat/import_all_modules.py:167: in main import_modules(modules) /usr/lib/rpm/redhat/import_all_modules.py:100: in import_modules importlib.import_module(module) /usr/lib64/python3.13/importlib/__init__.py:88: in import_module return _bootstrap._gcd_import(name[level:], package, level) :1387: in _gcd_import ??? :1360: in _find_and_load ??? :1331: in _find_and_load_unlocked ??? :935: in _load_unlocked ??? :1022: in exec_module ??? :488: in _call_with_frames_removed ??? _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ """Object-oriented filesystem paths. This module provides classes to represent abstract paths and concrete paths with operations that have semantics appropriate for different operating systems. """ from ._abc import * from ._local import * > __all__ = (_abc.__all__ + _local.__all__) E NameError: name '_abc' is not defined. Did you forget to import '_abc'? /usr/lib64/python3.13/pathlib/__init__.py:11: NameError ----------------------------- Captured stderr call ----------------------------- Check import: math Check import: wave Check import: csv Check import: pathlib --- tests/test_import_all_modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 71feeff..8b89bc4 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -160,15 +160,15 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('csv\npathlib\n') + test_file_2.write_text('csv\nnetrc\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'netrc', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('csv', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'netrc', 'logging'): assert module in sys.modules From 840a26c515db6b45fda5bce35422db3a7c2f09d5 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Wed, 10 Apr 2024 09:33:39 +0200 Subject: [PATCH 05/22] Add option -a to include BuilArch: noarch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Option -A disables that (the default, does nothing at the moment) Co-Authored-By: Miro Hrončok --- macros.python-srpm | 15 +++++++++++++-- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index d57e43f..00098d2 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -225,15 +225,19 @@ end } -%python_extras_subpkg(n:i:f:F) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' local option_F = '-F (skip %%files section)' + local option_a = '-a (insert BuildArch: noarch)' + local option_A = '-A (do not insert BuildArch: noarch (default))' local value_n = rpm.expand('%{-n*}') local value_i = rpm.expand('%{-i*}') local value_f = rpm.expand('%{-f*}') local value_F = rpm.expand('%{-F}') + local value_a = rpm.expand('%{-a}') + local value_A = rpm.expand('%{-A}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -250,6 +254,9 @@ if value_f ~= '' and value_F ~= '' then rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') end + if value_a ~= '' and value_A ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_a .. ' and ' .. option_A .. ' options are not possible}') + end if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end @@ -277,7 +284,11 @@ elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end - for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do + local tags = summary .. '\\\n' .. requires + if value_a ~= '' then + tags = tags .. '\\\nBuildArch: noarch' + end + for i, line in ipairs({pkgdef, tags, description, files, ''}) do print(line .. '\\\n') end end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ffd4d9a..7d3d1ef 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jun 25 2024 Cristian Le - 3.13-2 +- %%python_extras_subpkg: Add option -a to include BuildArch: noarch + * Thu Jun 06 2024 Karolina Surma - 3.13-1 - Update main Python to 3.13 diff --git a/tests/test_evals.py b/tests/test_evals.py index 1f953e0..e9fa44a 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -619,6 +619,45 @@ def test_python_extras_subpkg_F(): assert lines == expected +def test_python_extras_subpkg_a(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + BuildArch: noarch + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_A(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_aA(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -A -F toml', + version='6', release='7', fails=True) + assert lines[0] == ('error: %python_extras_subpkg: simultaneous -a ' + '(insert BuildArch: noarch) and -A (do not insert ' + 'BuildArch: noarch (default)) options are not possible') + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From d9e31f78970f00cbb27136960c602845ace55ea2 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 19 Jul 2024 15:32:23 +0000 Subject: [PATCH 06/22] Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 7d3d1ef..a2e5ca0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + * Tue Jun 25 2024 Cristian Le - 3.13-2 - %%python_extras_subpkg: Add option -a to include BuildArch: noarch From ef265656475b091f40dae1d8c229a03257f25bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 17:19:32 +0200 Subject: [PATCH 07/22] Stop testing Python 2.7 on the CI, we don't have it any more --- tests/pythontest.spec | 13 +++---------- tests/tests.yml | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 32bb039..26eaa2c 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,9 +1,8 @@ %global basedir /opt/test/byte_compilation # We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 -# Test with a representative of each. +# Test with a representative of each, except 2.7 which we no longer have %global python36_sitelib /usr/lib/python3.6/site-packages -%global python27_sitelib /usr/lib/python2.7/site-packages Name: pythontest Version: 0 @@ -12,7 +11,6 @@ Summary: ... License: MIT BuildRequires: python3-devel BuildRequires: python3.6 -BuildRequires: python2.7 %description ... @@ -33,23 +31,19 @@ echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py mkdir -p %{buildroot}%{python36_sitelib}/directory/ echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py -mkdir -p %{buildroot}%{python27_sitelib}/directory/ -echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py - %check LOCATIONS=" %{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/ %{buildroot}%{python36_sitelib}/directory/ - %{buildroot}%{python27_sitelib}/directory/ " # Count .py and .pyc files PY=$(find $LOCATIONS -name "*.py" | wc -l) PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) -# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7) -test $PY -eq 5 +# We should have 4 .py files (3 for python3, one for 3.6) +test $PY -eq 4 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files @@ -70,7 +64,6 @@ test $PY -ge $INODES %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py -%{python27_sitelib}/directory/file.py* %changelog diff --git a/tests/tests.yml b/tests/tests.yml index 4b8b2dd..af2c2a1 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -36,5 +36,4 @@ - python3-devel - python3-pytest - python3.6 - - python2.7 From 2c230bfcac1e97875f47d01e40364a55f44038e1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 18 Jan 2025 19:00:07 +0000 Subject: [PATCH 08/22] Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a2e5ca0..bcae6d1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild + * Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild From a3ba11ea648fda3b08613176a448b93a53386d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Fri, 14 Feb 2025 10:14:05 +0100 Subject: [PATCH 09/22] Add BuildRoot Policy script to modify the content of .dist-info/INSTALLER file Fixes: rhbz#2345186 --- brp-python-rpm-in-distinfo | 15 +++++++++++++ macros.python-srpm | 11 +++++++--- python-rpm-macros.spec | 9 +++++++- tests/test_rpm_in_distinfo.py | 41 +++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100755 brp-python-rpm-in-distinfo create mode 100644 tests/test_rpm_in_distinfo.py diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo new file mode 100755 index 0000000..473af00 --- /dev/null +++ b/brp-python-rpm-in-distinfo @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu +# If using normal root, avoid changing anything. +if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then + exit 0 +fi + +find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do + if cmp -s <(echo pip) "$installer" ; then + echo "rpm" > "$installer" + rm -f $(dirname "$installer")/RECORD + fi +done +exit 0 diff --git a/macros.python-srpm b/macros.python-srpm index 00098d2..f26ff0c 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -66,6 +66,8 @@ ### BRP scripts (and related macros) +## Modifies installation method in .dist-info/INSTALLER file to rpm +%python_rpm_in_distinfo 1 ## Automatically compile python files %py_auto_byte_compile 1 ## Should python bytecompilation errors terminate a build? @@ -78,16 +80,19 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts +%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink ## This macro is included in redhat-rpm-config's %%__os_install_post # Note that the order matters: -# 1. brp-python-bytecompile can create (or replace) pyc files -# 2. brp-fix-pyc-reproducibility can modify the pyc files from above -# 3. brp-python-hardlink de-duplicates identical pyc files +# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file +# 2. brp-python-bytecompile can create (or replace) pyc files +# 3. brp-fix-pyc-reproducibility can modify the pyc files from above +# 4. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ + %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bcae6d1..189c94a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -33,6 +33,8 @@ Source402: brp-python-hardlink # This one is from redhat-rpm-config < 190 # It has no upstream yet Source403: brp-fix-pyc-reproducibility +# brp script to write "rpm" string into the .dist-info/INSTALLER file +Source404: brp-python-rpm-in-distinfo # macros and lua: MIT # import_all_modules.py: MIT @@ -53,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -136,6 +138,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} %global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} %global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} +%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} %check @@ -156,6 +159,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -163,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 +- Add brp script to modify .dist-info/INSTALLER file + * Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py new file mode 100644 index 0000000..7546dc0 --- /dev/null +++ b/tests/test_rpm_in_distinfo.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import os +import pytest +import subprocess + +@pytest.fixture +def create_test_files(tmp_path): + def _create(subpath, installer_content): + dir_path = tmp_path / subpath + dir_path.mkdir(parents=True, exist_ok=True) + installer_file = dir_path / "INSTALLER" + installer_file.write_text(installer_content) + record_file = dir_path / "RECORD" + record_file.write_text("dummy content in RECORD file\n") + return dir_path + return _create + +testdata = [ + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), +] +@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) +def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, +path, installer_content, expected_installer_content, record_file_exists): + script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") + tmp_dir = create_test_files(path, installer_content) + monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) + result = subprocess.run( + [script_path], + capture_output=True, text=True + ) + + assert result.returncode == 0 + assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content + assert Path(tmp_dir / "RECORD").exists() is record_file_exists + From 4ddd2ea298c11703dd97252462270cadc6a438c7 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Fri, 9 Aug 2019 16:30:43 +0200 Subject: [PATCH 10/22] Eliminate use of ambiguous logical operators in script conditionals Prefer '[] && []' to '[ -a ]' and '[] || []' to '[ -o ]' in tests. -a and -o to mean AND and OR in a [ .. ] test expression is not well defined, and can cause incorrect results when arguments start with dashes or contain !. Moreover binary -a and -o are inherently ambiguous. test(1) man page recommends to use 'test EXPR1 && test EXPR2' or 'test EXPR1 || test EXPR2' instead. It corrects warnings [SC2166] spotted by covscan. --- brp-python-bytecompile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 1f7c4cd..b1e5db5 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi From 888775f1c5505bded0e8e8f2baf93b145454840e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 28 May 2025 10:32:43 +0200 Subject: [PATCH 11/22] Switch default Python version to 3.14 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f26ff0c..7ad62d1 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.13 +%__default_python3_version 3.14 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 189c94a..9faa80e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed May 28 2025 Karolina Surma - 3.14-1 +- Update main Python to 3.14 + * Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 - Add brp script to modify .dist-info/INSTALLER file From a74d2bb5c9a69b7b90b8afc5560f8ed4e95b2232 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 18:35:59 -0700 Subject: [PATCH 12/22] Minor style fixes suggested by ShellCheck. Mostly, these consist of preferring '[[' to '[' in bash scripts. Other changes include quoting unquoted variables, and explicitly specifying bash as the interpreter for scripts that use features not defined in POSIX sh Fixes SC2046, SC3001, and SC2292 --- brp-fix-pyc-reproducibility | 4 ++-- brp-python-bytecompile | 20 ++++++++++---------- brp-python-rpm-in-distinfo | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 16613b0..05857b3 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,7 +1,7 @@ #!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then +if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then exit 0 fi @@ -10,7 +10,7 @@ fi path_to_fix=${1:?} # First, check that the parser is available: -if [ ! -x /usr/bin/marshalparser ]; then +if [[ ! -x /usr/bin/marshalparser ]]; then echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" exit 1 fi diff --git a/brp-python-bytecompile b/brp-python-bytecompile index b1e5db5..fd7172f 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -6,7 +6,7 @@ errors_terminate=$2 # Therefore $1 ($default_python) is not needed and is invoked with "" by default. # $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 -if [ 0$extra -eq 1 ]; then +if [[ 0"$extra" -eq 1 ]]; then echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr exit 1 fi @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then +if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then exit 0 fi @@ -45,14 +45,14 @@ function python_bytecompile() # # Python 3.4 and higher # - if [ "$python_version" -ge 34 ]; then + if [[ "$python_version" -ge 34 ]]; then # We compile all opt levels in one go: only when $options is empty. - if [ -n "$options" ]; then + if [[ -n "$options" ]]; then return fi - if [ "$python_version" -ge 39 ]; then + if [[ "$python_version" -ge 39 ]]; then # For Pyhon 3.9+, use the standard library compileall_module=compileall else @@ -60,7 +60,7 @@ function python_bytecompile() compileall_module=compileall2 fi - if [ "$python_version" -ge 37 ]; then + if [[ "$python_version" -ge 37 ]]; then # Force the TIMESTAMP invalidation mode invalidation_option=--invalidation-mode=timestamp else @@ -69,7 +69,7 @@ function python_bytecompile() invalidation_option= fi - [ ! -z $exclude ] && exclude="-x '$exclude'" + [[ -n "$exclude" ]] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo index 473af00..b72d704 100755 --- a/brp-python-rpm-in-distinfo +++ b/brp-python-rpm-in-distinfo @@ -1,15 +1,15 @@ -#!/bin/sh +#!/usr/bin/bash set -eu # If using normal root, avoid changing anything. -if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then +if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then exit 0 fi find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do if cmp -s <(echo pip) "$installer" ; then echo "rpm" > "$installer" - rm -f $(dirname "$installer")/RECORD + rm -f "$(dirname "$installer")/RECORD" fi done exit 0 From 6b1cf3771c4370e61592d076fb8d313a5cff2519 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 22:35:06 -0700 Subject: [PATCH 13/22] The "exclude" variable in python_bytecompile is no longer used, so remove it. This resolves ShellCheck SC2089 and SC2090 warnings. --- brp-python-bytecompile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index fd7172f..1911fa1 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -36,7 +36,7 @@ function python_bytecompile() { local options=$1 local python_binary=$2 - local exclude=$3 + # local exclude=$3 # No longer used local python_libdir="$4" local compileall_flags="$5" @@ -69,18 +69,14 @@ function python_bytecompile() invalidation_option= fi - [[ -n "$exclude" ]] && exclude="-x '$exclude'" - # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels # -q disables verbose output # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -96,13 +92,10 @@ python_libdir = "$python_libdir" depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" -exclude = r"$exclude" class Filter: def search(self, path): ret = not os.path.realpath(path).startswith(build_root) - if exclude: - ret = ret or re.search(exclude, path) return ret sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) From b8a5807572e25276e83fbdc2f2a2d26f7e4592a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 16:55:23 +0200 Subject: [PATCH 14/22] Deprecate %py3_build, %py3_build_wheel, and %py3_install ...as well as their %py_... counterparts. https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros --- macros.python | 23 ++++++++++++++++++++--- macros.python3 | 6 +++--- python-rpm-macros.spec | 7 ++++++- tests/test_evals.py | 34 +++++++++++++++++++++++++++++++++- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/macros.python b/macros.python index 0ba4d78..4bd0da6 100644 --- a/macros.python +++ b/macros.python @@ -23,6 +23,23 @@ end print(_python_macro_cache[cache_key][name]) } +# Deprecation wrapper, warns only once per macro +# Options: +# -n - The name of the macro that is deprecated +%_python_deprecated(n:) %{lua: +if not _python_deprecated_warned then + -- This is intentionally a global lua table + _python_deprecated_warned = {} +end +if not _python_deprecated_warned[opt.n] then + _python_deprecated_warned[opt.n] = true + local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. + "See the current Python packaging guidelines: " .. + "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" + macros.warn({msg}) +end +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) @@ -69,17 +86,17 @@ print(_python_macro_cache[cache_key][name]) # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{expand:\\\ +%py_build() %{_python_deprecated -n py_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_wheel() %{expand:\\\ +%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{expand:\\\ +%py_install() %{_python_deprecated -n py_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python3 b/macros.python3 index 7197433..38cdcd4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -43,17 +43,17 @@ # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{expand:\\\ +%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_wheel() %{expand:\\\ +%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{expand:\\\ +%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 9faa80e..af7188c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -167,6 +167,11 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Jun 29 2025 Miro Hrončok - 3.14-2 +- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install +- Deprecate %%py_build, %%py_build_wheel, and %%py_install +- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros + * Wed May 28 2025 Karolina Surma - 3.14-1 - Update main Python to 3.14 diff --git a/tests/test_evals.py b/tests/test_evals.py index e9fa44a..b26ff29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -760,6 +760,9 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) + # strip the deprecation message + if 'deprecated' in lines[0]: + lines = lines[1:] assert lines[0] == ( 'error: attempt to use unversioned python, ' 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' @@ -777,7 +780,9 @@ def test_unversioned_python_errors(macro): @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + unverisoned = rpm_eval(macro, __python='/usr/bin/python3') + expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] + assert unverisoned == expected # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -958,3 +963,30 @@ def test_multi_python3(alt_x_y): lines = rpm_eval(evals) lines = [l for l in lines if l] # strip empty lines generated by %global assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +@pytest.mark.parametrize('macro', [ + '%py3_build', + '%py3_build_wheel', + '%py3_install', +]) +def test_deprecation(macro): + lines = rpm_eval(macro) + assert "is deprecated" in lines[0] + assert f"{macro} " in lines[0] + + +def test_multiple_deprecation(): + source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' + lines = rpm_eval(source) + + assert "is deprecated" in lines[0] + assert "%py3_build " in lines[0] + + assert "is deprecated" in lines[1] + assert "%py3_build_wheel " in lines[1] + + assert "is deprecated" in lines[2] + assert "%py3_install " in lines[2] + + assert "is deprecated" not in '\n'.join(lines[3:]) From 5cdc4d85a735c77daaa339f128f69ef98e9b0171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Huguet?= Date: Mon, 21 Jul 2025 08:07:36 +0200 Subject: [PATCH 15/22] pathfix.py: Don't fail on symbolic links The script ignores symlinks for obvious reasons. Don't return an error code in that case, though, as it makes the rpm builds fail if there are symlinks in /usr/bin and `pathfix.py ... /usr/bin/*` is used (e.g. via %pyproject_install from pyproject-rpm-macros). --- pathfix.py | 1 - python-rpm-macros.spec | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pathfix.py b/pathfix.py index 1d7db3a..6808382 100644 --- a/pathfix.py +++ b/pathfix.py @@ -56,7 +56,6 @@ def main(): if recursedown(arg): bad = 1 elif os.path.islink(arg): err(arg + ': will not process symbolic links\n') - bad = 1 else: if fix(arg): bad = 1 sys.exit(bad) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index af7188c..859d687 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 +- pathfix.py: Don't fail on symbolic links + * Sun Jun 29 2025 Miro Hrončok - 3.14-2 - Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install - Deprecate %%py_build, %%py_build_wheel, and %%py_install From 066459f836121418875c01b7f8bbc59980fe7be1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 25 Jul 2025 10:14:58 +0000 Subject: [PATCH 16/22] Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 859d687..a3a8be5 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 25 2025 Fedora Release Engineering +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + * Mon Jul 21 2025 Íñigo Huguet - 3.14-3 - pathfix.py: Don't fail on symbolic links From bfd1bc9738592449f8ddd6308c3d1b32c10243aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zachar?= Date: Fri, 25 Jul 2025 18:19:58 +0200 Subject: [PATCH 17/22] Drop STI and use tmt instead Resolves: rhbz#2383044 --- .fmf/version | 1 + plan.fmf | 35 +++++++++++++++++++++++++++++++++++ tests/tests.yml | 39 --------------------------------------- 3 files changed, 36 insertions(+), 39 deletions(-) create mode 100644 .fmf/version create mode 100644 plan.fmf delete mode 100644 tests/tests.yml diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/plan.fmf b/plan.fmf new file mode 100644 index 0000000..a681cdf --- /dev/null +++ b/plan.fmf @@ -0,0 +1,35 @@ +execute: + how: tmt + +discover: + - name: same_repo + how: shell + tests: + - name: pytest + test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - name: manual_byte_compilation_clamp_mtime_off + path: /tests + test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - name: manual_byte_compilation_clamp_mtime_on + path: /tests + test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - name: rpmlint_clamp_mtime_off + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: rpmlint_clamp_mtime_on + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + +prepare: + - name: Install dependencies + how: install + package: + - rpm-build + - rpmlint + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-pytest + - python3.6 + - dnf + - name: Update packages + how: shell + script: dnf upgrade -y diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index af2c2a1..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - tags: - - classic - tasks: - - dnf: - name: "*" - state: latest - -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - tests: - - pytest: - dir: . - run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation_clamp_mtime_off: - dir: . - run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - - manual_byte_compilation_clamp_mtime_on: - dir: . - run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec - - rpmlint_clamp_mtime_off: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - rpmlint_clamp_mtime_on: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - required_packages: - - rpm-build - - rpmlint - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-pytest - - python3.6 - From 364d99f4e1bd27fedeec5a1e8a7008f5bf5b5302 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 5 Aug 2025 20:16:56 +0200 Subject: [PATCH 18/22] import_all_modules: Add error handling for import failures Continue checking all modules when imports fail and provide detailed error reporting. Exit with proper status code when failures occur. --- import_all_modules.py | 21 ++++++++++- python-rpm-macros.spec | 5 ++- tests/test_import_all_modules.py | 64 +++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/import_all_modules.py b/import_all_modules.py index 3930236..97e924c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,6 +7,7 @@ import os import re import site import sys +import traceback from contextlib import contextmanager from pathlib import Path @@ -93,11 +94,24 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. + + Return a list of failed modules. ''' + failed_modules = [] + for module in modules: print('Check import:', module, file=sys.stderr) - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception: + traceback.print_exc(file=sys.stderr) + failed_modules.append(module) + + if failed_modules: + print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) + + return failed_modules def argparser(): @@ -164,7 +178,10 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - import_modules(modules) + failed_modules = import_modules(modules) + + if failed_modules: + raise SystemExit(1) if __name__ == '__main__': diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3a8be5..d536935 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Aug 11 2025 Lumír Balhar - 3.14-5 +- import_all_modules: Add error handling for import failures + * Fri Jul 25 2025 Fedora Release Engineering - Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 8b89bc4..dd10689 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['this_is_a_module_in_cwd']) @@ -175,7 +175,7 @@ def test_modules_from_files_are_found(tmp_path): def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,24 +253,70 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_module_with_error_propagates_exception(tmp_path): +def test_import_module_returns_failed_modules(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + assert failed_modules == ['this_is_a_module_in_tmp_path_2'] + + +def test_module_with_error_propagates_exception(tmp_path, capsys): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - # The correct exception must be raised - with pytest.raises(ZeroDivisionError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) + _, err = capsys.readouterr() + assert "ZeroDivisionError" in err + + +def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) + assert failed_modules == [] + + +def test_all_modules_are_imported(tmp_path, capsys): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + with pytest.raises(SystemExit): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'missing_module', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ]) + _, err = capsys.readouterr() + for i in range(1, 4): + assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err + assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): From 7d4cb5437d4f813b20dc2be44fe5d6517f48464f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 22 Jul 2025 20:51:50 +0200 Subject: [PATCH 19/22] Introduce %python_wheel_inject_sbom See https://discuss.python.org/t/encoding-origin-in-wheel-and-dist-info-metadata-for-downstream-security-backports/97436 --- macros.python-wheel-sbom | 124 +++++++++++++++++++++++++++++++++++++++ plan.fmf | 5 ++ python-rpm-macros.spec | 7 ++- tests/testwheel.spec | 102 ++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 macros.python-wheel-sbom create mode 100644 tests/testwheel.spec diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom new file mode 100644 index 0000000..a21460a --- /dev/null +++ b/macros.python-wheel-sbom @@ -0,0 +1,124 @@ +# The macros in this file are used to add SBOM to wheel files that we ship. +# Majority of Python packages will not need to do that, +# as they only use wheels as an intermediate artifact. +# The macros will be used by packages installing wheel to %%python_wheel_dir +# or by Python interpreters bundling their own (patched) wheels. +# +# The runtime dependencies are not Required by the python-rpm-macros package, +# users of this macro need to specify them on their own or rely on the fact that +# they are all available in the default buildroot. +# +# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS +# +# The wheels are modified in-place. + + +# Path of the SBOM file in the PEP 770 .dist-info/sboms directory +# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ +# section Recognized file patterns +%__python_wheel_sbom_filename bom.json + + +# The SBOM content to put to the file +# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 +%__python_wheel_sbom_content %{expand:{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "%{name}", + "version": "%{version}-%{release}", + "purl": "%{__python_wheel_purl}" + } + ] +}} + + +# The purl used above +# We use the src package name (which is easier to get and more useful to consumers). +# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 +# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ +%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src + + +# The purl namespace used above +# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ +# Distributors, define %%dist_purl_namespace to set this. +# The rest of the code is fallback for distributions without it (relying on %%dist_name). +%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: + if macros.epel then + -- being epel beats the %%dist_name value + -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 + print("epel") + else + local dist_map = { + -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm + -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 + ["Fedora Linux"] = "fedora", + -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 + ["CentOS Stream"] = "centos", + -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ + ["Red Hat Enterprise Linux"] = "redhat", + -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html + ["AlmaLinux"] = "almalinux", + -- from https://github.com/google/osv.dev/pull/2939 + ["Rocky Linux"] = "rocky-linux", + } + print(dist_map[macros.dist_name] or "unknown") + end +}} + + +# A Bash scriptlet to inject the SBOM file into the wheel(s) +# The macro takes positional nargs+ with wheel paths +# For each wheel, it +# 1. aborts if the SBOM file is already there (it won't override) +# 2. inserts the SBOM file to .dist-info/sboms +# 3. amends .dist-info/RECORD with the added SBOM file +%python_wheel_inject_sbom() %{expand:( + %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] + + set -eu -o pipefail + export LANG=C.utf-8 + + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + pwd0=$(pwd) + ret=0 + + for whl in %{*}; do + cd "$tmpdir" + if [[ "$whl" != /* ]]; then + whl="$pwd0/$whl" + fi + + record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + distinfo="${record%%/RECORD}" + bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" + + if zipinfo -1 "$whl" | grep -qFx "$bom"; then + echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 + ret=1 + continue + fi + + unzip "$whl" "$record" + mkdir "$distinfo/sboms" + echo '%{__python_wheel_sbom_content}' > "$bom" + checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" + size="$(wc --bytes "$bom" | cut -f1 -d' ')" + echo "$bom,$checksum,$size" >> "$record" + + if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then + touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" + fi + + zip -r "$whl" "$record" "$bom" + rm -rf "$distinfo" + cd "$pwd0" + done + + exit $ret +)} + diff --git a/plan.fmf b/plan.fmf index a681cdf..88850dd 100644 --- a/plan.fmf +++ b/plan.fmf @@ -17,6 +17,9 @@ discover: test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - name: rpmlint_clamp_mtime_on test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: python_wheel_inject_sbom + path: /tests + test: rpmbuild -ba testwheel.spec prepare: - name: Install dependencies @@ -27,6 +30,8 @@ prepare: - python-rpm-macros - python3-rpm-macros - python3-devel + - python3-setuptools + - python3-pip - python3-pytest - python3.6 - dnf diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d536935..86d2865 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -8,6 +8,7 @@ Source101: macros.python Source102: macros.python-srpm Source104: macros.python3 Source105: macros.pybytecompile +Source106: macros.python-wheel-sbom # Lua files Source201: python.lua @@ -55,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -149,6 +150,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/pathfix.py @@ -167,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed Aug 13 2025 Miro Hrončok - 3.14-6 +- Introduce %%python_wheel_inject_sbom + * Mon Aug 11 2025 Lumír Balhar - 3.14-5 - import_all_modules: Add error handling for import failures diff --git a/tests/testwheel.spec b/tests/testwheel.spec new file mode 100644 index 0000000..2e761fc --- /dev/null +++ b/tests/testwheel.spec @@ -0,0 +1,102 @@ +Name: testwheel +Epoch: 42 +Version: 1 +Release: 0%{?dist} +Summary: ... +License: MIT +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools >= 61 +BuildRequires: python3-pip + +%description +This builds and installs a wheel which we can then use as a test for +%%python_wheel_inject_sbom. + + +%prep +cat > pyproject.toml << EOF +[project] +name = "testwheel" +version = "1" + +[build-system] +requires = ["setuptools >= 61"] +build-backend = "setuptools.build_meta" +EOF + + +%build +export PIP_CONFIG_FILE=/dev/null +%{python3} -m pip wheel . --no-build-isolation + +# The macro should happily alter multiple wheels, let's make more +for i in {1..5}; do + mkdir ${i} + cp -a *.whl ${i} +done + +# using relative paths should succeed +%python_wheel_inject_sbom {1..5}/*.whl + +# repetitive use should bail out and fail (SBOM is already there) +%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true + +# each wheel should already have it, all should fail individually as well +for i in {1..5}; do + %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true +done + + +%install +mkdir -p %{buildroot}%{python_wheel_dir} +cp -a *.whl %{buildroot}%{python_wheel_dir} + +# using absolute paths should work +%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl + +# and fail when repeated +%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true + + +%check +%define venvsite venv/lib/python%{python3_version}/site-packages +%{python3} -m venv venv +venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl + +test -f %{venvsite}/testwheel-1.dist-info/RECORD +test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD +# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) +sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' + +# this deliberately uses a different mechanism than the macro +# if you are running this test on a different distro, adjust it +%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} + +PYTHONOPTIMIZE=0 %{python3} -c " +import json +with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: + sbom = json.load(fp) +assert len(sbom['components']) == 1 +assert sbom['components'][0]['type'] == 'library' +assert sbom['components'][0]['name'] == 'testwheel' +assert sbom['components'][0]['version'] == '1-0%{?dist}' +assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' +" + +# replace the installation with the original unaltered wheel +venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl +test -f %{venvsite}/testwheel-1.dist-info/RECORD +# no SBOM +test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true + + +%files +%{python_wheel_dir}/*.whl + + +%changelog +* Wed Aug 13 2025 Miro Hrončok - 42:1-0 +- A static changelog with a date, so we can clamp mtimes From 47de23b3c094fafcace77131f4ce6c233f5d1c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 29 Aug 2025 13:46:23 +0200 Subject: [PATCH 20/22] %python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos In python-setuptools-wheel, the macro was confused by setuptools/_vendor/autocommand-2.2.2.dist-info/RECORD (or other vendored RECORDs) --- macros.python-wheel-sbom | 2 +- python-rpm-macros.spec | 5 ++++- tests/testwheel.spec | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom index a21460a..41389f9 100644 --- a/macros.python-wheel-sbom +++ b/macros.python-wheel-sbom @@ -93,7 +93,7 @@ whl="$pwd0/$whl" fi - record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') distinfo="${record%%/RECORD}" bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 86d2865..ae3d558 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -169,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Aug 29 2025 Miro Hrončok - 3.14-7 +- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos + * Wed Aug 13 2025 Miro Hrončok - 3.14-6 - Introduce %%python_wheel_inject_sbom diff --git a/tests/testwheel.spec b/tests/testwheel.spec index 2e761fc..461d7f3 100644 --- a/tests/testwheel.spec +++ b/tests/testwheel.spec @@ -23,7 +23,18 @@ version = "1" [build-system] requires = ["setuptools >= 61"] build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["testwheel*"] EOF +# create a secondary dist-info folder in the project +# we need to ensure this file is not altered +mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info +touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +echo 'recursive-include testwheel/_vendor *' > MANIFEST.in %build @@ -70,6 +81,9 @@ grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/ # a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true + # this deliberately uses a different mechanism than the macro # if you are running this test on a different distro, adjust it %define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} From 9e5c1461f2346d245a18e3650834d548647dff48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 9 Sep 2025 15:00:10 +0200 Subject: [PATCH 21/22] %python_extras_subpkg: Add -v option to specify the required version(-release) This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package). --- macros.python-srpm | 6 ++++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 7ad62d1..f575868 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -230,7 +230,7 @@ end } -%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' @@ -243,6 +243,7 @@ local value_F = rpm.expand('%{-F}') local value_a = rpm.expand('%{-a}') local value_A = rpm.expand('%{-A}') + local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -265,7 +266,8 @@ if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ae3d558..032c913 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Sep 09 2025 Miro Hrončok - 3.14-8 +- %%python_extras_subpkg: Add -v option to specify the required version(-release) +- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) + * Fri Aug 29 2025 Miro Hrončok - 3.14-7 - %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos diff --git a/tests/test_evals.py b/tests/test_evals.py index b26ff29..31fb75d 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -658,6 +658,21 @@ def test_python_extras_subpkg_aA(): 'BuildArch: noarch (default)) options are not possible') +def test_python_extras_subpkg_v(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 1.2.3 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From 0d417402dbc243c6fa3170fff3e8d6b5b59638c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Oct 2025 16:10:53 +0200 Subject: [PATCH 22/22] %python_extras_subpkg: Only %ghost the egg-info/dist-info directory, not the content That way, accidentally unpackaged files within are reported as errors. Currently, when %python_extras_subpkg is used, the egg-info/dist-info directory is packaged as %ghost. When the main package does not have it, the RPM build would succeed. The extras packages would have the python3dist() requires and provides, but the main package would not. By adding %dir after %ghost, we only package the directory (which is enough for python3-rpm-generators to process it), but the files in the directory are not included. When not packaged in the main package, the RPM build fails. This is a safeguard against packaging mistakes. The visible difference is that rpm -ql/repoquery -l would only return the metadata directory. And the RPM build would fail if .egg-info is a file, which is only possible with Python < 3.12 for packages using distutils (no extras anyway). --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f575868..d5cd4e5 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -287,7 +287,7 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 032c913..263853a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Oct 16 2025 Miro Hrončok - 3.14-9 +- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content +- That way, accidentally unpackaged files within are reported as errors + * Tue Sep 09 2025 Miro Hrončok - 3.14-8 - %%python_extras_subpkg: Add -v option to specify the required version(-release) - This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) diff --git a/tests/test_evals.py b/tests/test_evals.py index 31fb75d..e6cc8f7 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -551,7 +551,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -562,7 +562,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected