From 86c391c493085ba7101aa595bf7c017b3dd52d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Oct 2022 17:34:50 +0200 Subject: [PATCH 01/44] Set PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus} from %pytest pytest-xdist 3+ respects this value when -n auto is used. See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/FQUUM3SZLRY3UUQJ355H4IJS2GRGIEVI/ --- macros.python3 | 1 + python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index c6aa17e..f45af14 100644 --- a/macros.python3 +++ b/macros.python3 @@ -113,4 +113,5 @@ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}\\\ %__pytest} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8a65425..8c4d3f3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -52,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -159,6 +159,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Nov 13 2022 Miro Hrončok - 3.11-6 +- Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest +- pytest-xdist 3+ respects this value when -n auto is used + * Tue Oct 25 2022 Lumír Balhar - 3.11-5 - Include pathfix.py in this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 94229dc..183dcc2 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -319,6 +319,11 @@ def test_pytest_command_suffix_alternate_pkgversion(version): assert f'/usr/bin/pytest-{version} -v' in lines[-1] +def test_pytest_sets_pytest_xdist_auto_num_workers(): + lines = rpm_eval('%pytest', _smp_build_ncpus=2) + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=2' in '\n'.join(lines) + + def test_pytest_undefined_addopts_are_not_set(): lines = rpm_eval('%pytest', __pytest_addopts=None) assert 'PYTEST_ADDOPTS' not in '\n'.join(lines) From b6479253006cef572fb0cbe7ce001b3048f127b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Oct 2022 17:47:59 +0200 Subject: [PATCH 02/44] Expose the environment variables used by %pytest via %{py3_test_envvars} This way, we will be able to reuse them in %tox from pyproject-rpm-macros. Packagers will be able to use them in their spec files. See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/A3QQSHP5OSLIVMCL52AR2GRBRXYQHU6B/ --- macros.python | 10 ++++++++++ macros.python3 | 13 ++++++++----- python-rpm-macros.spec | 1 + tests/test_evals.py | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/macros.python b/macros.python index 65cf929..b45c985 100644 --- a/macros.python +++ b/macros.python @@ -131,6 +131,16 @@ end } +# Environment variables for testing used standalone, e.g.: +# %%{py_test_envvars} %%{python} -m unittest +%py_test_envvars %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + %python_disable_dependency_generator() \ %undefine __pythondist_requires \ %{nil} diff --git a/macros.python3 b/macros.python3 index f45af14..6aa541b 100644 --- a/macros.python3 +++ b/macros.python3 @@ -105,13 +105,16 @@ end } -# This is intended for Python 3 only, hence also no Python version in the name. -%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) -%pytest %{expand:\\\ +# Environment variables used by %%pytest, %%tox or standalone, e.g.: +# %%{py3_test_envvars} %%{python3} -m unittest +%py3_test_envvars %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}\\\ - %__pytest} + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) +%pytest %py3_test_envvars %__pytest diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8c4d3f3..e7844fc 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -162,6 +162,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest - pytest-xdist 3+ respects this value when -n auto is used +- Expose the environment variables used by %%pytest via %%{py3_test_envvars} * Tue Oct 25 2022 Lumír Balhar - 3.11-5 - Include pathfix.py in this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 183dcc2..96ac705 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -357,6 +357,28 @@ def test_pytest_addopts_preserves_envvar(__pytest_addopts): assert 'z--' not in echoed +@pytest.mark.parametrize('__pytest_addopts', ['-X', None]) +def test_py3_test_envvars(lib, __pytest_addopts): + lines = rpm_eval('%{py3_test_envvars}\\\n%{python3} -m unittest', + buildroot='BUILDROOT', + _smp_build_ncpus='3', + __pytest_addopts=__pytest_addopts) + assert all(l.endswith('\\') for l in lines[:-1]) + stripped_lines = [l.strip(' \\') for l in lines] + sitearch = f'BUILDROOT/usr/{lib}/python{X_Y}/site-packages' + sitelib = f'BUILDROOT/usr/lib/python{X_Y}/site-packages' + assert f'PYTHONPATH="${{PYTHONPATH:-{sitearch}:{sitelib}}}"' in stripped_lines + assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines + assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines + assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=3' in stripped_lines + if __pytest_addopts: + assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines + else: + assert 'PYTEST_ADDOPTS' not in ''.join(lines) + assert stripped_lines[-1] == '/usr/bin/python3 -m unittest' + + def test_pypi_source_default_name(): urls = rpm_eval('%pypi_source', name='foo', version='6') @@ -691,6 +713,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%py_install_egg', '%py_install_wheel', '%py_check_import', + '%py_test_envvars', ]) From e3494799f813e6d73646597d7730bf128a4c94ab Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 14 Dec 2022 15:37:02 +0100 Subject: [PATCH 03/44] Test data: use csv which will not be removed from stdlib soon --- tests/test_import_all_modules.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 52e1d7e..71feeff 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -141,15 +141,15 @@ def test_modules_from_sys_path_found(tmp_path): def test_modules_from_file_are_found(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' - test_file.write_text('math\nwave\nsunau\n') + test_file.write_text('math\nwave\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau'): + for m in ('math', 'wave', 'csv'): sys.modules.pop(m, None) modules_main(['-f', str(test_file)]) - assert 'sunau' in sys.modules + assert 'csv' in sys.modules assert 'math' in sys.modules assert 'wave' in sys.modules @@ -160,15 +160,15 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('sunau\npathlib\n') - test_file_3.write_text('logging\nsunau\n') + test_file_2.write_text('csv\npathlib\n') + test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'pathlib', 'logging'): assert module in sys.modules From eb7a4fda28cadd4ae5d374414a57b2620a8fe341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 12 Dec 2022 11:13:46 +0100 Subject: [PATCH 04/44] Bytecompilation: Unset $SOURCE_DATE_EPOCH when %clamp_mtime_to_source_date_epoch is not set https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode --- macros.pybytecompile | 6 +++--- macros.python-srpm | 5 ++++- python-rpm-macros.spec | 12 ++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index dd8b495..9e5505b 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -18,7 +18,7 @@ %py_byte_compile()\ py2_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ failure=0\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ @@ -27,13 +27,13 @@ py2_byte_compile () {\ }\ \ py3_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ diff --git a/macros.python-srpm b/macros.python-srpm index 7104f7f..70a04ac 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -73,9 +73,12 @@ ## Should python bytecompilation compile outside python specific directories? ## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 %_python_bytecompile_extra 0 +## Helper macro to unset $SOURCE_DATE_EPOCH if %%clamp_mtime_to_source_date_epoch is not set +## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode +%__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e7844fc..2191707 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -52,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -130,9 +130,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ # It also ensures that: # - our BRPs can execute # - if our BRPs affect this package, we don't need to build it twice -%global __brp_python_bytecompile %{buildroot}%{__brp_python_bytecompile} -%global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink} -%global __brp_fix_pyc_reproducibility %{buildroot}%{__brp_fix_pyc_reproducibility} +%define add_buildroot() %{lua:print((macros[macros[1]]:gsub(macros._rpmconfigdir, macros.buildroot .. macros._rpmconfigdir)))} +%global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} +%global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} +%global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} %check @@ -159,6 +160,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Dec 19 2022 Miro Hrončok - 3.11-7 +- Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set + * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest - pytest-xdist 3+ respects this value when -n auto is used From e4baf5ab7e10485fb909bbb715f3243d14a5e752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Dec 2022 20:24:53 +0100 Subject: [PATCH 05/44] Bytecompilation: Pass --invalidation-mode=timestamp to compileall (Only on Python 3.7+, where it exists and matters.) This will replace patch 328 in Python and fix https://bugzilla.redhat.com/2133850 --- brp-python-bytecompile | 11 ++++++++++- macros.pybytecompile | 14 +++++++++++--- python-rpm-macros.spec | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index c1749ab..347e789 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -47,6 +47,15 @@ function python_bytecompile() compileall_module=compileall2 fi + if [ "$python_version" -ge 37 ]; then + # Force the TIMESTAMP invalidation mode + invalidation_option=--invalidation-mode=timestamp + else + # For older Pythons, the option does not exist + # as the invalidation is always based on size+mtime + invalidation_option= + fi + [ ! -z $exclude ] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib @@ -58,7 +67,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # diff --git a/macros.pybytecompile b/macros.pybytecompile index 9e5505b..9c9da85 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -26,16 +26,21 @@ py2_byte_compile () {\ test $failure -eq 0\ }\ \ -py3_byte_compile () {\ +py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ +py37_byte_compile () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ +}\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ @@ -44,10 +49,13 @@ py39_byte_compile () {\ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ # and it was merged back to stdlib in Python >= 3.9 \ +# Only Python 3.7+ supports and needs the --invalidation-mode option \ if [ "$python_version" -ge 39 ]; then \ py39_byte_compile "%1" "%2"; \ +elif [ "$python_version" -ge 37 ]; then \ +py37_byte_compile "%1" "%2"; \ elif [ "$python_version" -ge 34 ]; then \ -py3_byte_compile "%1" "%2"; \ +py34_byte_compile "%1" "%2"; \ else \ py2_byte_compile "%1" "%2"; \ fi diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 2191707..8ca30d6 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -162,6 +162,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set +- Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From 77912c744dcfd7e4ab7d833686719640bcf45e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Dec 2022 21:06:17 +0100 Subject: [PATCH 06/44] Add a script to clamp source mtimes, invoke it from the bytecompilation BRP/macro https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes --- brp-python-bytecompile | 15 ++++ clamp_source_mtime.py | 161 +++++++++++++++++++++++++++++++++++++++++ macros.pybytecompile | 8 ++ python-rpm-macros.spec | 6 +- 4 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 clamp_source_mtime.py diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 347e789..3092bdf 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -16,6 +16,16 @@ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi +# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +function python_clamp_source_mtime() +{ + local _=$1 + local python_binary=$2 + local _=$3 + local python_libdir="$4" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir" +} + # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. @@ -123,6 +133,11 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had inaccessible mtime + exit 1 + fi python_bytecompile "" "$python_binary" "" "$python_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py new file mode 100644 index 0000000..7349ea3 --- /dev/null +++ b/clamp_source_mtime.py @@ -0,0 +1,161 @@ +"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH + +When called as a script with arguments, this compiles the directories +given as arguments recursively. + +If upstream is interested, this can be later integrated to the compileall module +as an additional option (e.g. --clamp-source-mtime). + +License: +This has been derived from the Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ +""" +import os +import sys + +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) + +__all__ = ["clamp_dir", "clamp_file"] + + +def _walk_dir(dir, maxlevels, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) + + +def clamp_dir(dir, source_date_epoch, quiet=0): + """Clamp the mtime of all modules in the given directory tree. + + Arguments: + + dir: the directory to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + maxlevels = sys.getrecursionlimit() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + for file in files: + if not clamp_file(file, source_date_epoch, quiet=quiet): + success = False + return success + + +def clamp_file(fullname, source_date_epoch, quiet=0): + """Clamp the mtime of one file. + + Arguments: + + fullname: the file to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + if os.path.isfile(fullname) and not os.path.islink(fullname): + if name[-3:] == '.py': + try: + mtime = int(os.stat(fullname).st_mtime) + atime = int(os.stat(fullname).st_atime) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error checking mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + if mtime > source_date_epoch: + if not quiet: + print('Clamping mtime of {!r}'.format(fullname)) + try: + os.utime(fullname, (atime, source_date_epoch)) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error clamping mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + return True + + +def main(): + """Script main program.""" + import argparse + + source_date_epoch = os.getenv('SOURCE_DATE_EPOCH') + if not source_date_epoch: + print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set") + return True # This is a success, no action needed + try: + source_date_epoch = int(source_date_epoch) + except ValueError: + print("$SOURCE_DATE_EPOCH must be an integer") + return False + + parser = argparse.ArgumentParser( + description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+', + help=('zero or more file and directory paths ' + 'to clamp')) + + args = parser.parse_args() + clamp_dests = args.clamp_dest + + success = True + try: + for dest in clamp_dests: + if os.path.isfile(dest): + if not clamp_file(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + else: + if not clamp_dir(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + return success + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/macros.pybytecompile b/macros.pybytecompile index 9c9da85..8c9fbee 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -17,6 +17,12 @@ # Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ +clamp_source_mtime () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \ +}\ +\ py2_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ @@ -45,6 +51,8 @@ py39_byte_compile () {\ \ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ +# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \ +clamp_source_mtime "%1" "%2"; \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8ca30d6..d9daf49 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -18,6 +18,7 @@ Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_ Source302: import_all_modules.py %global pathfix_version 1.0.0 Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py +Source304: clamp_source_mtime.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -35,7 +36,7 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py: PSFv2 +# compileall2.py, clamp_source_mtime.py: PSFv2 # pathfix.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -120,6 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -150,6 +152,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/clamp_source_mtime.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -163,6 +166,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) +- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From cb1dbcc44b105f3deadc99f0b79b7711bc0ff6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 14:38:05 +0100 Subject: [PATCH 07/44] Add Python 2.7 support to clamp_source_mtime.py This script is also invoked with Python 2 --- clamp_source_mtime.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py index 7349ea3..1d03a6b 100644 --- a/clamp_source_mtime.py +++ b/clamp_source_mtime.py @@ -10,6 +10,7 @@ License: This has been derived from the Python's compileall module and it follows Python licensing. For more info see: https://www.python.org/psf/license/ """ +from __future__ import print_function import os import sys @@ -41,8 +42,9 @@ def _walk_dir(dir, maxlevels, quiet=0): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): - yield from _walk_dir(fullname, maxlevels=maxlevels - 1, - quiet=quiet) + for result in _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet): + yield result def clamp_dir(dir, source_date_epoch, quiet=0): From 18ceb6caef878e0de8b22297a63161423653684d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 14:42:54 +0100 Subject: [PATCH 08/44] CI tests: Build pythontest.spec with and without %clamp_mtime_to_source_date_epoch --- tests/pythontest.spec | 5 +++++ tests/tests.yml | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 3b707c5..50c136a 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -71,3 +71,8 @@ test $PY -ge $INODES %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py %{python27_sitelib}/directory/file.py* + + +%changelog +* Thu Jan 01 2015 Fedora Packager - 0-0 +- This changelog entry exists and is deliberately set in the past diff --git a/tests/tests.yml b/tests/tests.yml index 11bb3ae..ac03d2e 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -16,9 +16,12 @@ - pytest: dir: . run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation: + - manual_byte_compilation_clamp_mtime_off: dir: . - run: rpmbuild -ba pythontest.spec + run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - manual_byte_compilation_clamp_mtime_on: + dir: . + run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec required_packages: - rpm-build - python-rpm-macros From 13c1c0c519fd78ced17f790149f5c050015dce45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 15:06:03 +0100 Subject: [PATCH 09/44] CI tests: Assert rpmlint's python-bytecode-inconsistent-mtime is not happening --- tests/pythontest.spec | 2 +- tests/tests.yml | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 50c136a..32bb039 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -7,7 +7,7 @@ Name: pythontest Version: 0 -Release: 0 +Release: 0%{?dist} Summary: ... License: MIT BuildRequires: python3-devel diff --git a/tests/tests.yml b/tests/tests.yml index ac03d2e..4b8b2dd 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -18,12 +18,19 @@ run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - manual_byte_compilation_clamp_mtime_off: dir: . - run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - manual_byte_compilation_clamp_mtime_on: dir: . - run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - rpmlint_clamp_mtime_off: + dir: . + run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - rpmlint_clamp_mtime_on: + dir: . + run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 required_packages: - rpm-build + - rpmlint - python-rpm-macros - python3-rpm-macros - python3-devel From a3ea23bd5ed0226fd03d49dc200939bfc778d73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 15:14:32 +0100 Subject: [PATCH 10/44] Bytecompilation: Compile Python files in parallel, according to %_smp_mflags --- brp-python-bytecompile | 11 +++++++---- macros.pybytecompile | 6 +++--- macros.python-srpm | 2 +- python-rpm-macros.spec | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 3092bdf..1f7c4cd 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -11,6 +11,8 @@ if [ 0$extra -eq 1 ]; then exit 1 fi +compileall_flags="$4" + # If using normal root, avoid changing anything. if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 @@ -36,6 +38,7 @@ function python_bytecompile() local python_binary=$2 local exclude=$3 local python_libdir="$4" + local compileall_flags="$5" python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") @@ -77,7 +80,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -133,12 +136,12 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. - python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had inaccessible mtime exit 1 fi - python_bytecompile "" "$python_binary" "" "$python_libdir" + python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 @@ -146,7 +149,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing - python_bytecompile "-O" "$python_binary" "" "$python_libdir" + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 diff --git a/macros.pybytecompile b/macros.pybytecompile index 8c9fbee..e5e5a47 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -35,18 +35,18 @@ py2_byte_compile () {\ py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ py37_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + $python_binary -s -B -m compileall %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ diff --git a/macros.python-srpm b/macros.python-srpm index 70a04ac..5e57102 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -78,7 +78,7 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_mflags}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d9daf49..919f470 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -167,6 +167,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) - Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +- Bytecompilation: Compile Python files in parallel, according to %%_smp_mflags * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From 9a36534a957594f370f269b739e28dd23b4de66b Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 20 Jan 2023 17:04:00 +0000 Subject: [PATCH 11/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 919f470..58a7704 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) From 3ca74ad94c19d316938a27cd5758b4ed5053c3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 6 Jan 2023 15:04:29 +0100 Subject: [PATCH 12/44] Memoize values of macros that execute python to get their value Macros like %{python3_sitelib} were evaluated at every instance in the spec (each time, Python was started, sysconfig was imported...). When there were many instances (40+), it might have taken more than a minute to parse the spec file. This way, the macros are defined via %global on first usage. Every repetitive usage reuses the actual value. --- macros.python | 18 +++++++++--------- macros.python3 | 16 ++++++++-------- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 14 ++++++++++++-- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/macros.python b/macros.python index b45c985..4568614 100644 --- a/macros.python +++ b/macros.python @@ -3,18 +3,18 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") +%python_sitelib %{global python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitelib} +%python_sitearch %{global python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitearch} +%python_version %{global python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python_version} +%python_version_nodots %{global python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python_version_nodots} +%python_platform %{global python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")}%{python_platform} +%python_platform_triplet %{global python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python_platform_triplet} +%python_ext_suffix %{global python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python_ext_suffix} +%python_cache_tag %{global python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")}%{python_cache_tag} %py_setup setup.py %_py_shebang_s s -%_py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_P %{global _py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py_shebang_P} %py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) diff --git a/macros.python3 b/macros.python3 index 6aa541b..0e4678e 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,18 +1,18 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitelib %{global python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python3_sitelib} %python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") +%python3_version %{global python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python3_version} +%python3_version_nodots %{global python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python3_version_nodots} +%python3_platform %{global python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")}%{python3_platform} +%python3_platform_triplet %{global python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python3_platform_triplet} +%python3_ext_suffix %{global python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python3_ext_suffix} +%python3_cache_tag %{global python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")}%{python3_cache_tag} %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %_py3_shebang_s s -%_py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_P %{global _py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py3_shebang_P} %py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 58a7704..abf5ba1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jan 20 2023 Miro Hrončok - 3.11-9 +- Memoize values of macros that execute python to get their value +- Fixes: rhbz#2155505 + * Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 - Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 96ac705..294faaa 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -720,8 +720,18 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) - assert lines == ['error: attempt to use unversioned python, ' - 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'] + assert lines[0] == ( + 'error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' + ) + # when the macros are %global, the error is longer + # we deliberately allow this extra line to be optional + if len(lines) > 1: + # the failed macro is not unnecessarily our tested macro + pattern = r'error: Macro %\S+ failed to expand' + assert re.match(pattern, lines[1]) + # but there should be no more lines + assert len(lines) < 3 @unversioned_macros From 10e0e5309d691e16d2e86d329827f47993879bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Mar 2023 19:43:17 +0100 Subject: [PATCH 13/44] Don't assume %_smp_mflags only ever contains -jX, use -j%_smp_build_ncpus directly When we added %_smp_mflags here, Petr Viktorin asked the question: https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/154#comment-124613 > I couldn't find docs for %_smp_mflags. > How much of a guarantee is there that it contains no other flags than -j? My answer was: > %_smp_mflags is documented in https://rpm-packaging-guide.github.io/ > and used in many other RPM macros in Fedora and upstream everywhere. > There is no official guarantee that it will never contain anything else, > but if it does, I assume multiple things would burn. > I am willing to take that risk. Turns out, the world did not burn, but packagers do set %_smp_mflags to -lX, which does not work with compileall. Fixes https://bugzilla.redhat.com/2179149 --- macros.pybytecompile | 6 +++--- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index e5e5a47..e81ddae 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -35,18 +35,18 @@ py2_byte_compile () {\ py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ py37_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + $python_binary -s -B -m compileall %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ diff --git a/macros.python-srpm b/macros.python-srpm index 5e57102..37cf09a 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -78,7 +78,7 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_mflags}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index abf5ba1..b53f7df 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 10%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Mar 16 2023 Miro Hrončok - 3.11-10 +- Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly +- Fixes: rhbz#2179149 + * Fri Jan 20 2023 Miro Hrončok - 3.11-9 - Memoize values of macros that execute python to get their value - Fixes: rhbz#2155505 From ab399410274763176a151ac3437f2cc7b31e8857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Oct 2022 17:47:59 +0200 Subject: [PATCH 14/44] Expose the environment variables used by %pytest via %{py3_test_envvars} This way, we will be able to reuse them in %tox from pyproject-rpm-macros. Packagers will be able to use them in their spec files. See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/A3QQSHP5OSLIVMCL52AR2GRBRXYQHU6B/ ... This is a backport of b6479253006cef572fb0cbe7ce001b3048f127b3 sans 86c391c493085ba7101aa595bf7c017b3dd52d06 which sets PYTEST_XDIST_AUTO_NUM_WORKERS. --- macros.python | 9 +++++++++ macros.python3 | 13 ++++++++----- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 22 ++++++++++++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/macros.python b/macros.python index 65cf929..c73e21d 100644 --- a/macros.python +++ b/macros.python @@ -131,6 +131,15 @@ end } +# Environment variables for testing used standalone, e.g.: +# %%{py_test_envvars} %%{python} -m unittest +%py_test_envvars %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}} + %python_disable_dependency_generator() \ %undefine __pythondist_requires \ %{nil} diff --git a/macros.python3 b/macros.python3 index c6aa17e..204b0de 100644 --- a/macros.python3 +++ b/macros.python3 @@ -105,12 +105,15 @@ end } -# This is intended for Python 3 only, hence also no Python version in the name. -%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) -%pytest %{expand:\\\ +# Environment variables used by %%pytest, %%tox or standalone, e.g.: +# %%{py3_test_envvars} %%{python3} -m unittest +%py3_test_envvars %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - %__pytest} + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}} + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) +%pytest %py3_test_envvars %__pytest diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8a65425..1726983 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -52,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -159,6 +159,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Nov 13 2022 Miro Hrončok - 3.11-6 +- Expose the environment variables used by %%pytest via %%{py3_test_envvars} + * Tue Oct 25 2022 Lumír Balhar - 3.11-5 - Include pathfix.py in this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 94229dc..8e5eb29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -352,6 +352,27 @@ def test_pytest_addopts_preserves_envvar(__pytest_addopts): assert 'z--' not in echoed +@pytest.mark.parametrize('__pytest_addopts', ['-X', None]) +def test_py3_test_envvars(lib, __pytest_addopts): + lines = rpm_eval('%{py3_test_envvars}\\\n%{python3} -m unittest', + buildroot='BUILDROOT', + _smp_build_ncpus='3', + __pytest_addopts=__pytest_addopts) + assert all(l.endswith('\\') for l in lines[:-1]) + stripped_lines = [l.strip(' \\') for l in lines] + sitearch = f'BUILDROOT/usr/{lib}/python{X_Y}/site-packages' + sitelib = f'BUILDROOT/usr/lib/python{X_Y}/site-packages' + assert f'PYTHONPATH="${{PYTHONPATH:-{sitearch}:{sitelib}}}"' in stripped_lines + assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines + assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines + assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines + if __pytest_addopts: + assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines + else: + assert 'PYTEST_ADDOPTS' not in ''.join(lines) + assert stripped_lines[-1] == '/usr/bin/python3 -m unittest' + + def test_pypi_source_default_name(): urls = rpm_eval('%pypi_source', name='foo', version='6') @@ -686,6 +707,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%py_install_egg', '%py_install_wheel', '%py_check_import', + '%py_test_envvars', ]) From 3c21a3a25818ac1512c891b0d3ab79d8fbcaa30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Tue, 13 Jun 2023 14:43:51 +0200 Subject: [PATCH 15/44] Python 3.12 https://fedoraproject.org/wiki/Changes/Python3.12 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 37cf09a..278571f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.11 +%__default_python3_version 3.12 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b53f7df..347fb4e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 10%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 +- Update main Python to Python 3.12 +- https://fedoraproject.org/wiki/Changes/Python3.12 + * Thu Mar 16 2023 Miro Hrončok - 3.11-10 - Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly - Fixes: rhbz#2179149 From f6368ebcaa89b656bbf95f266ed83c841fbbc77d Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 21 Jul 2023 13:45:20 +0000 Subject: [PATCH 16/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 347fb4e..3948428 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + * Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 - Update main Python to Python 3.12 - https://fedoraproject.org/wiki/Changes/Python3.12 From 933da64fe645d29d8782b7c4df613bdcb2c3b6a8 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 9 Aug 2023 11:08:17 +0200 Subject: [PATCH 17/44] Declare the license as an SPDX expression --- python-rpm-macros.spec | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3948428..8220091 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -36,10 +36,10 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py, clamp_source_mtime.py: PSFv2 -# pathfix.py: PSFv2 -# brp scripts: GPLv2+ -License: MIT and Python and GPLv2+ +# compileall2.py, clamp_source_mtime.py: PSF-2.0 +# pathfix.py: PSF-2.0 +# brp scripts: GPL-2.0-or-later +License: MIT AND PSF-2.0 AND GPL-2.0-or-later # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed Aug 09 2023 Karolina Surma - 3.11-3 +- Declare the license as an SPDX expression + * Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild From d4a26d93d0939ab246c2fd894c66d4c3a6c0ed9c Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Tue, 5 Sep 2023 00:42:40 -0500 Subject: [PATCH 18/44] Remove %py3_build_egg and %py3_install_egg macros. %py3_install_egg is nonfunctional; setuptools removed the easy_install entrypoint years ago. %py3_build_egg is technically functional but has been superseded by newer macros. Calling setup.py directly is deprecated, and building/installing eggs to begin with is deprecated. The macros are not used in any Fedora packages and are broken. It's time to remove them. --- macros.python | 11 ----------- macros.python3 | 11 ----------- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 2 -- 4 files changed, 4 insertions(+), 25 deletions(-) diff --git a/macros.python b/macros.python index 4568614..e058fff 100644 --- a/macros.python +++ b/macros.python @@ -33,11 +33,6 @@ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - %py_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} @@ -49,12 +44,6 @@ rm -rfv %{buildroot}%{_bindir}/__pycache__ } -%py_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python_sitelib} - %{__python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - %py_install_wheel() %{expand:\\\ %{__python} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python3 b/macros.python3 index 0e4678e..06a7db2 100644 --- a/macros.python3 +++ b/macros.python3 @@ -31,11 +31,6 @@ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - %py3_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} @@ -47,12 +42,6 @@ rm -rfv %{buildroot}%{_bindir}/__pycache__ } -%py3_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python3_sitelib} - %{__python3} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - %py3_install_wheel() %{expand:\\\ %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8220091..708cd30 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Sep 05 2023 Maxwell G - 3.12-4 +- Remove %%py3_build_egg and %%py3_install_egg macros. + * Wed Aug 09 2023 Karolina Surma - 3.11-3 - Declare the license as an SPDX expression diff --git a/tests/test_evals.py b/tests/test_evals.py index 294faaa..2e0e6e1 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -707,10 +707,8 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_cache_tag', '%py_shebang_fix', '%py_build', - '%py_build_egg', '%py_build_wheel', '%py_install', - '%py_install_egg', '%py_install_wheel', '%py_check_import', '%py_test_envvars', From c765382ec8ef1f9c67e1745bc03c9374ddc6a15a Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Fri, 1 Sep 2023 16:22:10 +0200 Subject: [PATCH 19/44] Fix the changelog entries --- python-rpm-macros.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 708cd30..322250d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -166,10 +166,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Tue Sep 05 2023 Maxwell G - 3.12-4 - Remove %%py3_build_egg and %%py3_install_egg macros. -* Wed Aug 09 2023 Karolina Surma - 3.11-3 +* Wed Aug 09 2023 Karolina Surma - 3.12-3 - Declare the license as an SPDX expression -* Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 +* Fri Jul 21 2023 Fedora Release Engineering - 3.12-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild * Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 From 5eec3f76022eb061a1a8d2488d48ec3ab477d04c Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Wed, 31 May 2023 17:05:04 -0500 Subject: [PATCH 20/44] Fix python macro memoizing to account for changing %__python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new `%_python_memoize` macro that caches the values of various python macros in a lua table based on the current value of `%__python` / `%__python3`. The Python macros are adjusted to use this macro for memoization instead of the current approach. This way, the macros will return values that apply to the _current_ `%__python3` value when packagers create multi-python specfiles that toggle the value of `%python3_pkgversion`. Relates: https://bugzilla.redhat.com/2209055 Co-Authored-By: Miro Hrončok --- macros.python | 59 +++++++++++++++++++++++++++++++++++------- macros.python3 | 35 ++++++++++++++++++------- python-rpm-macros.spec | 5 +++- tests/test_evals.py | 35 +++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 21 deletions(-) diff --git a/macros.python b/macros.python index e058fff..0d29ae0 100644 --- a/macros.python +++ b/macros.python @@ -1,20 +1,61 @@ +# Memoize a macro to avoid calling the same expensive code multiple times in +# the specfile. +# There is no error handling, +# memoizing an undefined macro (or using such a key) has undefined behavior. +# Options: +# -n - The name of the macro to wrap +# -k - The name of the macro to use as a cache key +%_python_memoize(n:k:) %{lua: +local name = opt.n +-- NB: We use rpm.expand() here instead of the macros table to make sure errors +-- are propogated properly. +local cache_key = rpm.expand("%{" .. opt.k .. "}") +if not _python_macro_cache then + -- This is intentionally a global lua table + _python_macro_cache = {} +end +if not _python_macro_cache[cache_key] then + _python_macro_cache[cache_key] = {} +end +if not _python_macro_cache[cache_key][name] then + _python_macro_cache[cache_key][name] = rpm.expand("%{" .. name .. "}") +end +print(_python_macro_cache[cache_key][name]) +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python_sitelib %{global python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitelib} -%python_sitearch %{global python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitearch} -%python_version %{global python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python_version} -%python_version_nodots %{global python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python_version_nodots} -%python_platform %{global python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")}%{python_platform} -%python_platform_triplet %{global python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python_platform_triplet} -%python_ext_suffix %{global python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python_ext_suffix} -%python_cache_tag %{global python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")}%{python_cache_tag} +%__python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitelib %{_python_memoize -n __python_sitelib -k __python} + +%__python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitearch %{_python_memoize -n __python_sitearch -k __python} + +%__python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version %{_python_memoize -n __python_version -k __python} + +%__python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_version_nodots %{_python_memoize -n __python_version_nodots -k __python} + +%__python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform %{_python_memoize -n __python_platform -k __python} + +%__python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_platform_triplet %{_python_memoize -n __python_platform_triplet -k __python} + +%__python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_ext_suffix %{_python_memoize -n __python_ext_suffix -k __python} + +%__python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") +%python_cache_tag %{_python_memoize -n __python_cache_tag -k __python} %py_setup setup.py %_py_shebang_s s -%_py_shebang_P %{global _py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py_shebang_P} +%__py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_P %{_python_memoize -n __py_shebang_P -k __python} %py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) diff --git a/macros.python3 b/macros.python3 index 06a7db2..a813557 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,18 +1,35 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python3_sitelib %{global python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python3_sitelib} -%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_version %{global python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python3_version} -%python3_version_nodots %{global python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python3_version_nodots} -%python3_platform %{global python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")}%{python3_platform} -%python3_platform_triplet %{global python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python3_platform_triplet} -%python3_ext_suffix %{global python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python3_ext_suffix} -%python3_cache_tag %{global python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")}%{python3_cache_tag} +%__python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitelib %{_python_memoize -n __python3_sitelib -k __python3} + +%__python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitearch %{_python_memoize -n __python3_sitearch -k __python3} + +%__python3_version %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version %{_python_memoize -n __python3_version -k __python3} + +%__python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_version_nodots %{_python_memoize -n __python3_version_nodots -k __python3} + +%__python3_platform %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python3_platform %{_python_memoize -n __python3_platform -k __python3} + +%__python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_platform_triplet %{_python_memoize -n __python3_platform_triplet -k __python3} + +%__python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_ext_suffix %{_python_memoize -n __python3_ext_suffix -k __python3} + +%__python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; print(sys.implementation.cache_tag)") +%python3_cache_tag %{_python_memoize -n __python3_cache_tag -k __python3} + %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %_py3_shebang_s s -%_py3_shebang_P %{global _py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py3_shebang_P} +%__py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_P %{_python_memoize -n __py3_shebang_P -k __python3} %py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 322250d..e8f7a4c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Oct 09 2023 Maxwell G - 3.12-5 +- Fix python macro memoizing to account for changing %%__python3 + * Tue Sep 05 2023 Maxwell G - 3.12-4 - Remove %%py3_build_egg and %%py3_install_egg macros. diff --git a/tests/test_evals.py b/tests/test_evals.py index 2e0e6e1..69f689e 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -22,6 +22,8 @@ TESTED_FILES = os.getenv("TESTED_FILES", None) def rpm_eval(expression, fails=False, **kwargs): + if isinstance(expression, str): + expression = [expression] cmd = ['rpmbuild'] if TESTED_FILES: cmd += ['--macros', TESTED_FILES] @@ -30,7 +32,8 @@ def rpm_eval(expression, fails=False, **kwargs): cmd += ['--undefine', var] else: cmd += ['--define', f'{var} {value}'] - cmd += ['--eval', expression] + for e in expression: + cmd += ['--eval', e] cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if fails: @@ -724,7 +727,7 @@ def test_unversioned_python_errors(macro): ) # when the macros are %global, the error is longer # we deliberately allow this extra line to be optional - if len(lines) > 1: + if len(lines) > 1 and "error: lua script failed" not in lines[1]: # the failed macro is not unnecessarily our tested macro pattern = r'error: Macro %\S+ failed to expand' assert re.match(pattern, lines[1]) @@ -888,3 +891,31 @@ def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_s # Compare the last line of the command, that's where lua part is evaluated expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' assert lines[-1].strip() == expected + + +def test_multi_python(alt_x_y): + """ + Ensure memoized %python_version works when switching %__python back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python /usr/bin/python{version}', '%python_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +def test_multi_python3(alt_x_y): + """ + Ensure memoized %python3_version works when switching %__python3 back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python3 /usr/bin/python{version}', '%python3_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] From d1c3ea93f88d6d4e83489373727104d322f00d42 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Mon, 22 Jan 2024 06:41:47 +0000 Subject: [PATCH 21/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e8f7a4c..8b5fba3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + * Mon Oct 09 2023 Maxwell G - 3.12-5 - Fix python macro memoizing to account for changing %%__python3 From b2f798fc819c1c0b3254d6cd4d24e04dd97e49cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 18 Jan 2023 16:36:05 +0100 Subject: [PATCH 22/44] %py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set --- macros.python | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/macros.python b/macros.python index 0d29ae0..0ba4d78 100644 --- a/macros.python +++ b/macros.python @@ -169,7 +169,7 @@ print(_python_macro_cache[cache_key][name]) PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} %python_disable_dependency_generator() \ %undefine __pythondist_requires \ diff --git a/macros.python3 b/macros.python3 index a813557..7197433 100644 --- a/macros.python3 +++ b/macros.python3 @@ -119,7 +119,7 @@ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} # This is intended for Python 3 only, hence also no Python version in the name. %__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8b5fba3..06fc622 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Jan 25 2024 Miro Hrončok - 3.12-7 +- %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set + * Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 - Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 69f689e..1f953e0 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -324,7 +324,7 @@ def test_pytest_command_suffix_alternate_pkgversion(version): def test_pytest_sets_pytest_xdist_auto_num_workers(): lines = rpm_eval('%pytest', _smp_build_ncpus=2) - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=2' in '\n'.join(lines) + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-2}"' in '\n'.join(lines) def test_pytest_undefined_addopts_are_not_set(): @@ -374,7 +374,7 @@ def test_py3_test_envvars(lib, __pytest_addopts): assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=3' in stripped_lines + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-3}"' in stripped_lines if __pytest_addopts: assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines else: From 2acd7cb9b37be66ac1fe1d85134698d66b6ae170 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 21 Mar 2024 17:02:42 +0100 Subject: [PATCH 23/44] Update bundled compileall2 to version 0.8.0 --- compileall2.py | 80 +++++++++++++++++++++++------------------- python-rpm-macros.spec | 7 ++-- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/compileall2.py b/compileall2.py index c58e545..ea7e76f 100644 --- a/compileall2.py +++ b/compileall2.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, if compiles all modules on sys.path, without +Without arguments, it compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -36,11 +36,11 @@ PY35 = sys.version_info[0:2] >= (3, 5) # introduced in Python 3.7. These cases are covered by variables here or by PY37 # variable itself. if PY37: - pyc_struct_format = '<4sll' + pyc_struct_format = '<4sLL' pyc_header_lenght = 12 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) else: - pyc_struct_format = '<4sl' + pyc_struct_format = '<4sL' pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) @@ -106,7 +106,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path @@ -120,23 +120,34 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, stripdir = dir prependdir = ddir ddir = None - if workers is not None: - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - elif workers != 1: - try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: - workers = 1 + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits + try: + _check_system_limits() + except NotImplementedError: + workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True - if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + if workers != 1 and ProcessPoolExecutor is not None: + mp_context_arg = {} + if PY37: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + mp_context_arg = {"mp_context": mp_context} + # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + **mp_context_arg) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -178,7 +189,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. @@ -190,10 +201,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) - else: - fullname = str(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None @@ -206,13 +215,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: @@ -258,7 +267,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack(*(pyc_header_format + (mtime,))) + expect = struct.pack(*(pyc_header_format + (mtime & 0xFFFF_FFFF,))) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(pyc_header_lenght) @@ -301,9 +310,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) + encoding = sys.stdout.encoding or sys.getdefaultencoding() + msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False @@ -408,8 +416,8 @@ def main(): type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses optimization level of ' - 'Python interpreter itself (specified by -O).')) + 'Default is -1 which uses the optimization level ' + 'of the Python interpreter itself (see -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', @@ -456,7 +464,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: @@ -464,9 +473,6 @@ def main(): print("Error reading file list {}".format(args.flist)) return False - if args.workers is not None: - args.workers = args.workers or None - if PY37 and args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 06fc622..a3c2511 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -13,7 +13,7 @@ Source105: macros.pybytecompile Source201: python.lua # Python code -%global compileall2_version 0.7.1 +%global compileall2_version 0.8.0 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py %global pathfix_version 1.0.0 @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Mar 22 2024 Lumír Balhar - 3.12-8 +- Update bundled compileall2 to version 0.8.0 + * Thu Jan 25 2024 Miro Hrončok - 3.12-7 - %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set From d211646a5e0cc32f62efe963d7970b896445f9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 1 Mar 2024 00:27:31 +0100 Subject: [PATCH 24/44] brp-fix-pyc-reproducibility: use more strict shell style 1. Error out on unset variables. 2. Implement suggestion made by shellcheck: use "||" instead of "-o"" in test. Shellcheck also suggested -print0 to avoid ambiguity with filenames with embedded newlines, but instead use '-exec {} +', which doesn't need xargs but still passes multiple arguments to a single marshalparser invocation. (Those issues are unlikely to cause problems in the rpm environment, but it's nice to be shellcheck-clean to use shellcheck during development.) Co-Authored-By: Benjamin A. Beasley --- brp-fix-pyc-reproducibility | 10 ++++------ python-rpm-macros.spec | 5 ++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 536a126..16613b0 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,13 +1,13 @@ -#!/bin/bash -e +#!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then exit 0 fi # Defined as %py_reproducible_pyc_path macro and passed here as # the first command-line argument -path_to_fix=$1 +path_to_fix=${1:?} # First, check that the parser is available: if [ ! -x /usr/bin/marshalparser ]; then @@ -15,6 +15,4 @@ if [ ! -x /usr/bin/marshalparser ]; then exit 1 fi -# Set pipefail so if $path_to_fix does not exist, the build fails -set -o pipefail -find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite +find "$path_to_fix" -type f -name '*.pyc' -exec /usr/bin/marshalparser --fix --overwrite '{}' '+' diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3c2511..5d2daf2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 +- Minor improvements to brp-fix-pyc-reproducibility + * Fri Mar 22 2024 Lumír Balhar - 3.12-8 - Update bundled compileall2 to version 0.8.0 From a389cc467425766a0122be2c143a9a4fdb76191e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 16 Oct 2023 16:45:02 +0200 Subject: [PATCH 25/44] Update main Python to 3.13 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 278571f..d57e43f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.12 +%__default_python3_version 3.13 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5d2daf2..ffd4d9a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Jun 06 2024 Karolina Surma - 3.13-1 +- Update main Python to 3.13 + * Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 - Minor improvements to brp-fix-pyc-reproducibility From d430201e60a2d911f58bb979b1eca6299bbc96c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 25 Jun 2024 12:34:11 +0200 Subject: [PATCH 26/44] CI tests fix for Python 3.13 We should not unimport pathlib when we use pathlib. This was always fragile. Choose a module we don't use: The failure: ______________________ test_modules_from_files_are_found _______________________ tmp_path = PosixPath('/tmp/pytest-of-root/pytest-0/test_modules_from_files_are_fo0') def test_modules_from_files_are_found(tmp_path): test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') test_file_2.write_text('csv\npathlib\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): sys.modules.pop(m, None) > modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) test_import_all_modules.py:170: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/rpm/redhat/import_all_modules.py:167: in main import_modules(modules) /usr/lib/rpm/redhat/import_all_modules.py:100: in import_modules importlib.import_module(module) /usr/lib64/python3.13/importlib/__init__.py:88: in import_module return _bootstrap._gcd_import(name[level:], package, level) :1387: in _gcd_import ??? :1360: in _find_and_load ??? :1331: in _find_and_load_unlocked ??? :935: in _load_unlocked ??? :1022: in exec_module ??? :488: in _call_with_frames_removed ??? _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ """Object-oriented filesystem paths. This module provides classes to represent abstract paths and concrete paths with operations that have semantics appropriate for different operating systems. """ from ._abc import * from ._local import * > __all__ = (_abc.__all__ + _local.__all__) E NameError: name '_abc' is not defined. Did you forget to import '_abc'? /usr/lib64/python3.13/pathlib/__init__.py:11: NameError ----------------------------- Captured stderr call ----------------------------- Check import: math Check import: wave Check import: csv Check import: pathlib --- tests/test_import_all_modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 71feeff..8b89bc4 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -160,15 +160,15 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('csv\npathlib\n') + test_file_2.write_text('csv\nnetrc\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'netrc', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('csv', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'netrc', 'logging'): assert module in sys.modules From 840a26c515db6b45fda5bce35422db3a7c2f09d5 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Wed, 10 Apr 2024 09:33:39 +0200 Subject: [PATCH 27/44] Add option -a to include BuilArch: noarch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Option -A disables that (the default, does nothing at the moment) Co-Authored-By: Miro Hrončok --- macros.python-srpm | 15 +++++++++++++-- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index d57e43f..00098d2 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -225,15 +225,19 @@ end } -%python_extras_subpkg(n:i:f:F) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' local option_F = '-F (skip %%files section)' + local option_a = '-a (insert BuildArch: noarch)' + local option_A = '-A (do not insert BuildArch: noarch (default))' local value_n = rpm.expand('%{-n*}') local value_i = rpm.expand('%{-i*}') local value_f = rpm.expand('%{-f*}') local value_F = rpm.expand('%{-F}') + local value_a = rpm.expand('%{-a}') + local value_A = rpm.expand('%{-A}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -250,6 +254,9 @@ if value_f ~= '' and value_F ~= '' then rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') end + if value_a ~= '' and value_A ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_a .. ' and ' .. option_A .. ' options are not possible}') + end if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end @@ -277,7 +284,11 @@ elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end - for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do + local tags = summary .. '\\\n' .. requires + if value_a ~= '' then + tags = tags .. '\\\nBuildArch: noarch' + end + for i, line in ipairs({pkgdef, tags, description, files, ''}) do print(line .. '\\\n') end end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ffd4d9a..7d3d1ef 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jun 25 2024 Cristian Le - 3.13-2 +- %%python_extras_subpkg: Add option -a to include BuildArch: noarch + * Thu Jun 06 2024 Karolina Surma - 3.13-1 - Update main Python to 3.13 diff --git a/tests/test_evals.py b/tests/test_evals.py index 1f953e0..e9fa44a 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -619,6 +619,45 @@ def test_python_extras_subpkg_F(): assert lines == expected +def test_python_extras_subpkg_a(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + BuildArch: noarch + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_A(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_aA(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -A -F toml', + version='6', release='7', fails=True) + assert lines[0] == ('error: %python_extras_subpkg: simultaneous -a ' + '(insert BuildArch: noarch) and -A (do not insert ' + 'BuildArch: noarch (default)) options are not possible') + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From d9e31f78970f00cbb27136960c602845ace55ea2 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 19 Jul 2024 15:32:23 +0000 Subject: [PATCH 28/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 7d3d1ef..a2e5ca0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + * Tue Jun 25 2024 Cristian Le - 3.13-2 - %%python_extras_subpkg: Add option -a to include BuildArch: noarch From ef265656475b091f40dae1d8c229a03257f25bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 17:19:32 +0200 Subject: [PATCH 29/44] Stop testing Python 2.7 on the CI, we don't have it any more --- tests/pythontest.spec | 13 +++---------- tests/tests.yml | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 32bb039..26eaa2c 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,9 +1,8 @@ %global basedir /opt/test/byte_compilation # We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 -# Test with a representative of each. +# Test with a representative of each, except 2.7 which we no longer have %global python36_sitelib /usr/lib/python3.6/site-packages -%global python27_sitelib /usr/lib/python2.7/site-packages Name: pythontest Version: 0 @@ -12,7 +11,6 @@ Summary: ... License: MIT BuildRequires: python3-devel BuildRequires: python3.6 -BuildRequires: python2.7 %description ... @@ -33,23 +31,19 @@ echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py mkdir -p %{buildroot}%{python36_sitelib}/directory/ echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py -mkdir -p %{buildroot}%{python27_sitelib}/directory/ -echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py - %check LOCATIONS=" %{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/ %{buildroot}%{python36_sitelib}/directory/ - %{buildroot}%{python27_sitelib}/directory/ " # Count .py and .pyc files PY=$(find $LOCATIONS -name "*.py" | wc -l) PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) -# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7) -test $PY -eq 5 +# We should have 4 .py files (3 for python3, one for 3.6) +test $PY -eq 4 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files @@ -70,7 +64,6 @@ test $PY -ge $INODES %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py -%{python27_sitelib}/directory/file.py* %changelog diff --git a/tests/tests.yml b/tests/tests.yml index 4b8b2dd..af2c2a1 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -36,5 +36,4 @@ - python3-devel - python3-pytest - python3.6 - - python2.7 From 2c230bfcac1e97875f47d01e40364a55f44038e1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 18 Jan 2025 19:00:07 +0000 Subject: [PATCH 30/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a2e5ca0..bcae6d1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild + * Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild From a3ba11ea648fda3b08613176a448b93a53386d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Fri, 14 Feb 2025 10:14:05 +0100 Subject: [PATCH 31/44] Add BuildRoot Policy script to modify the content of .dist-info/INSTALLER file Fixes: rhbz#2345186 --- brp-python-rpm-in-distinfo | 15 +++++++++++++ macros.python-srpm | 11 +++++++--- python-rpm-macros.spec | 9 +++++++- tests/test_rpm_in_distinfo.py | 41 +++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100755 brp-python-rpm-in-distinfo create mode 100644 tests/test_rpm_in_distinfo.py diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo new file mode 100755 index 0000000..473af00 --- /dev/null +++ b/brp-python-rpm-in-distinfo @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu +# If using normal root, avoid changing anything. +if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then + exit 0 +fi + +find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do + if cmp -s <(echo pip) "$installer" ; then + echo "rpm" > "$installer" + rm -f $(dirname "$installer")/RECORD + fi +done +exit 0 diff --git a/macros.python-srpm b/macros.python-srpm index 00098d2..f26ff0c 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -66,6 +66,8 @@ ### BRP scripts (and related macros) +## Modifies installation method in .dist-info/INSTALLER file to rpm +%python_rpm_in_distinfo 1 ## Automatically compile python files %py_auto_byte_compile 1 ## Should python bytecompilation errors terminate a build? @@ -78,16 +80,19 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts +%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink ## This macro is included in redhat-rpm-config's %%__os_install_post # Note that the order matters: -# 1. brp-python-bytecompile can create (or replace) pyc files -# 2. brp-fix-pyc-reproducibility can modify the pyc files from above -# 3. brp-python-hardlink de-duplicates identical pyc files +# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file +# 2. brp-python-bytecompile can create (or replace) pyc files +# 3. brp-fix-pyc-reproducibility can modify the pyc files from above +# 4. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ + %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bcae6d1..189c94a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -33,6 +33,8 @@ Source402: brp-python-hardlink # This one is from redhat-rpm-config < 190 # It has no upstream yet Source403: brp-fix-pyc-reproducibility +# brp script to write "rpm" string into the .dist-info/INSTALLER file +Source404: brp-python-rpm-in-distinfo # macros and lua: MIT # import_all_modules.py: MIT @@ -53,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -136,6 +138,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} %global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} %global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} +%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} %check @@ -156,6 +159,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -163,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 +- Add brp script to modify .dist-info/INSTALLER file + * Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py new file mode 100644 index 0000000..7546dc0 --- /dev/null +++ b/tests/test_rpm_in_distinfo.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import os +import pytest +import subprocess + +@pytest.fixture +def create_test_files(tmp_path): + def _create(subpath, installer_content): + dir_path = tmp_path / subpath + dir_path.mkdir(parents=True, exist_ok=True) + installer_file = dir_path / "INSTALLER" + installer_file.write_text(installer_content) + record_file = dir_path / "RECORD" + record_file.write_text("dummy content in RECORD file\n") + return dir_path + return _create + +testdata = [ + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), +] +@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) +def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, +path, installer_content, expected_installer_content, record_file_exists): + script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") + tmp_dir = create_test_files(path, installer_content) + monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) + result = subprocess.run( + [script_path], + capture_output=True, text=True + ) + + assert result.returncode == 0 + assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content + assert Path(tmp_dir / "RECORD").exists() is record_file_exists + From 4ddd2ea298c11703dd97252462270cadc6a438c7 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Fri, 9 Aug 2019 16:30:43 +0200 Subject: [PATCH 32/44] Eliminate use of ambiguous logical operators in script conditionals Prefer '[] && []' to '[ -a ]' and '[] || []' to '[ -o ]' in tests. -a and -o to mean AND and OR in a [ .. ] test expression is not well defined, and can cause incorrect results when arguments start with dashes or contain !. Moreover binary -a and -o are inherently ambiguous. test(1) man page recommends to use 'test EXPR1 && test EXPR2' or 'test EXPR1 || test EXPR2' instead. It corrects warnings [SC2166] spotted by covscan. --- brp-python-bytecompile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 1f7c4cd..b1e5db5 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi From 888775f1c5505bded0e8e8f2baf93b145454840e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 28 May 2025 10:32:43 +0200 Subject: [PATCH 33/44] Switch default Python version to 3.14 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f26ff0c..7ad62d1 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.13 +%__default_python3_version 3.14 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 189c94a..9faa80e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed May 28 2025 Karolina Surma - 3.14-1 +- Update main Python to 3.14 + * Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 - Add brp script to modify .dist-info/INSTALLER file From a74d2bb5c9a69b7b90b8afc5560f8ed4e95b2232 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 18:35:59 -0700 Subject: [PATCH 34/44] Minor style fixes suggested by ShellCheck. Mostly, these consist of preferring '[[' to '[' in bash scripts. Other changes include quoting unquoted variables, and explicitly specifying bash as the interpreter for scripts that use features not defined in POSIX sh Fixes SC2046, SC3001, and SC2292 --- brp-fix-pyc-reproducibility | 4 ++-- brp-python-bytecompile | 20 ++++++++++---------- brp-python-rpm-in-distinfo | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 16613b0..05857b3 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,7 +1,7 @@ #!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then +if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then exit 0 fi @@ -10,7 +10,7 @@ fi path_to_fix=${1:?} # First, check that the parser is available: -if [ ! -x /usr/bin/marshalparser ]; then +if [[ ! -x /usr/bin/marshalparser ]]; then echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" exit 1 fi diff --git a/brp-python-bytecompile b/brp-python-bytecompile index b1e5db5..fd7172f 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -6,7 +6,7 @@ errors_terminate=$2 # Therefore $1 ($default_python) is not needed and is invoked with "" by default. # $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 -if [ 0$extra -eq 1 ]; then +if [[ 0"$extra" -eq 1 ]]; then echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr exit 1 fi @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then +if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then exit 0 fi @@ -45,14 +45,14 @@ function python_bytecompile() # # Python 3.4 and higher # - if [ "$python_version" -ge 34 ]; then + if [[ "$python_version" -ge 34 ]]; then # We compile all opt levels in one go: only when $options is empty. - if [ -n "$options" ]; then + if [[ -n "$options" ]]; then return fi - if [ "$python_version" -ge 39 ]; then + if [[ "$python_version" -ge 39 ]]; then # For Pyhon 3.9+, use the standard library compileall_module=compileall else @@ -60,7 +60,7 @@ function python_bytecompile() compileall_module=compileall2 fi - if [ "$python_version" -ge 37 ]; then + if [[ "$python_version" -ge 37 ]]; then # Force the TIMESTAMP invalidation mode invalidation_option=--invalidation-mode=timestamp else @@ -69,7 +69,7 @@ function python_bytecompile() invalidation_option= fi - [ ! -z $exclude ] && exclude="-x '$exclude'" + [[ -n "$exclude" ]] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo index 473af00..b72d704 100755 --- a/brp-python-rpm-in-distinfo +++ b/brp-python-rpm-in-distinfo @@ -1,15 +1,15 @@ -#!/bin/sh +#!/usr/bin/bash set -eu # If using normal root, avoid changing anything. -if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then +if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then exit 0 fi find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do if cmp -s <(echo pip) "$installer" ; then echo "rpm" > "$installer" - rm -f $(dirname "$installer")/RECORD + rm -f "$(dirname "$installer")/RECORD" fi done exit 0 From 6b1cf3771c4370e61592d076fb8d313a5cff2519 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 22:35:06 -0700 Subject: [PATCH 35/44] The "exclude" variable in python_bytecompile is no longer used, so remove it. This resolves ShellCheck SC2089 and SC2090 warnings. --- brp-python-bytecompile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index fd7172f..1911fa1 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -36,7 +36,7 @@ function python_bytecompile() { local options=$1 local python_binary=$2 - local exclude=$3 + # local exclude=$3 # No longer used local python_libdir="$4" local compileall_flags="$5" @@ -69,18 +69,14 @@ function python_bytecompile() invalidation_option= fi - [[ -n "$exclude" ]] && exclude="-x '$exclude'" - # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels # -q disables verbose output # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -96,13 +92,10 @@ python_libdir = "$python_libdir" depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" -exclude = r"$exclude" class Filter: def search(self, path): ret = not os.path.realpath(path).startswith(build_root) - if exclude: - ret = ret or re.search(exclude, path) return ret sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) From b8a5807572e25276e83fbdc2f2a2d26f7e4592a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 16:55:23 +0200 Subject: [PATCH 36/44] Deprecate %py3_build, %py3_build_wheel, and %py3_install ...as well as their %py_... counterparts. https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros --- macros.python | 23 ++++++++++++++++++++--- macros.python3 | 6 +++--- python-rpm-macros.spec | 7 ++++++- tests/test_evals.py | 34 +++++++++++++++++++++++++++++++++- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/macros.python b/macros.python index 0ba4d78..4bd0da6 100644 --- a/macros.python +++ b/macros.python @@ -23,6 +23,23 @@ end print(_python_macro_cache[cache_key][name]) } +# Deprecation wrapper, warns only once per macro +# Options: +# -n - The name of the macro that is deprecated +%_python_deprecated(n:) %{lua: +if not _python_deprecated_warned then + -- This is intentionally a global lua table + _python_deprecated_warned = {} +end +if not _python_deprecated_warned[opt.n] then + _python_deprecated_warned[opt.n] = true + local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. + "See the current Python packaging guidelines: " .. + "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" + macros.warn({msg}) +end +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) @@ -69,17 +86,17 @@ print(_python_macro_cache[cache_key][name]) # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{expand:\\\ +%py_build() %{_python_deprecated -n py_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_wheel() %{expand:\\\ +%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{expand:\\\ +%py_install() %{_python_deprecated -n py_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python3 b/macros.python3 index 7197433..38cdcd4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -43,17 +43,17 @@ # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{expand:\\\ +%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_wheel() %{expand:\\\ +%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{expand:\\\ +%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 9faa80e..af7188c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -167,6 +167,11 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Jun 29 2025 Miro Hrončok - 3.14-2 +- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install +- Deprecate %%py_build, %%py_build_wheel, and %%py_install +- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros + * Wed May 28 2025 Karolina Surma - 3.14-1 - Update main Python to 3.14 diff --git a/tests/test_evals.py b/tests/test_evals.py index e9fa44a..b26ff29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -760,6 +760,9 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) + # strip the deprecation message + if 'deprecated' in lines[0]: + lines = lines[1:] assert lines[0] == ( 'error: attempt to use unversioned python, ' 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' @@ -777,7 +780,9 @@ def test_unversioned_python_errors(macro): @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + unverisoned = rpm_eval(macro, __python='/usr/bin/python3') + expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] + assert unverisoned == expected # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -958,3 +963,30 @@ def test_multi_python3(alt_x_y): lines = rpm_eval(evals) lines = [l for l in lines if l] # strip empty lines generated by %global assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +@pytest.mark.parametrize('macro', [ + '%py3_build', + '%py3_build_wheel', + '%py3_install', +]) +def test_deprecation(macro): + lines = rpm_eval(macro) + assert "is deprecated" in lines[0] + assert f"{macro} " in lines[0] + + +def test_multiple_deprecation(): + source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' + lines = rpm_eval(source) + + assert "is deprecated" in lines[0] + assert "%py3_build " in lines[0] + + assert "is deprecated" in lines[1] + assert "%py3_build_wheel " in lines[1] + + assert "is deprecated" in lines[2] + assert "%py3_install " in lines[2] + + assert "is deprecated" not in '\n'.join(lines[3:]) From 5cdc4d85a735c77daaa339f128f69ef98e9b0171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Huguet?= Date: Mon, 21 Jul 2025 08:07:36 +0200 Subject: [PATCH 37/44] pathfix.py: Don't fail on symbolic links The script ignores symlinks for obvious reasons. Don't return an error code in that case, though, as it makes the rpm builds fail if there are symlinks in /usr/bin and `pathfix.py ... /usr/bin/*` is used (e.g. via %pyproject_install from pyproject-rpm-macros). --- pathfix.py | 1 - python-rpm-macros.spec | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pathfix.py b/pathfix.py index 1d7db3a..6808382 100644 --- a/pathfix.py +++ b/pathfix.py @@ -56,7 +56,6 @@ def main(): if recursedown(arg): bad = 1 elif os.path.islink(arg): err(arg + ': will not process symbolic links\n') - bad = 1 else: if fix(arg): bad = 1 sys.exit(bad) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index af7188c..859d687 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 +- pathfix.py: Don't fail on symbolic links + * Sun Jun 29 2025 Miro Hrončok - 3.14-2 - Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install - Deprecate %%py_build, %%py_build_wheel, and %%py_install From 066459f836121418875c01b7f8bbc59980fe7be1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 25 Jul 2025 10:14:58 +0000 Subject: [PATCH 38/44] Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 859d687..a3a8be5 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 25 2025 Fedora Release Engineering +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + * Mon Jul 21 2025 Íñigo Huguet - 3.14-3 - pathfix.py: Don't fail on symbolic links From bfd1bc9738592449f8ddd6308c3d1b32c10243aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zachar?= Date: Fri, 25 Jul 2025 18:19:58 +0200 Subject: [PATCH 39/44] Drop STI and use tmt instead Resolves: rhbz#2383044 --- .fmf/version | 1 + plan.fmf | 35 +++++++++++++++++++++++++++++++++++ tests/tests.yml | 39 --------------------------------------- 3 files changed, 36 insertions(+), 39 deletions(-) create mode 100644 .fmf/version create mode 100644 plan.fmf delete mode 100644 tests/tests.yml diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/plan.fmf b/plan.fmf new file mode 100644 index 0000000..a681cdf --- /dev/null +++ b/plan.fmf @@ -0,0 +1,35 @@ +execute: + how: tmt + +discover: + - name: same_repo + how: shell + tests: + - name: pytest + test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - name: manual_byte_compilation_clamp_mtime_off + path: /tests + test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - name: manual_byte_compilation_clamp_mtime_on + path: /tests + test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - name: rpmlint_clamp_mtime_off + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: rpmlint_clamp_mtime_on + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + +prepare: + - name: Install dependencies + how: install + package: + - rpm-build + - rpmlint + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-pytest + - python3.6 + - dnf + - name: Update packages + how: shell + script: dnf upgrade -y diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index af2c2a1..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - tags: - - classic - tasks: - - dnf: - name: "*" - state: latest - -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - tests: - - pytest: - dir: . - run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation_clamp_mtime_off: - dir: . - run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - - manual_byte_compilation_clamp_mtime_on: - dir: . - run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec - - rpmlint_clamp_mtime_off: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - rpmlint_clamp_mtime_on: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - required_packages: - - rpm-build - - rpmlint - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-pytest - - python3.6 - From 364d99f4e1bd27fedeec5a1e8a7008f5bf5b5302 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 5 Aug 2025 20:16:56 +0200 Subject: [PATCH 40/44] import_all_modules: Add error handling for import failures Continue checking all modules when imports fail and provide detailed error reporting. Exit with proper status code when failures occur. --- import_all_modules.py | 21 ++++++++++- python-rpm-macros.spec | 5 ++- tests/test_import_all_modules.py | 64 +++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/import_all_modules.py b/import_all_modules.py index 3930236..97e924c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,6 +7,7 @@ import os import re import site import sys +import traceback from contextlib import contextmanager from pathlib import Path @@ -93,11 +94,24 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. + + Return a list of failed modules. ''' + failed_modules = [] + for module in modules: print('Check import:', module, file=sys.stderr) - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception: + traceback.print_exc(file=sys.stderr) + failed_modules.append(module) + + if failed_modules: + print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) + + return failed_modules def argparser(): @@ -164,7 +178,10 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - import_modules(modules) + failed_modules = import_modules(modules) + + if failed_modules: + raise SystemExit(1) if __name__ == '__main__': diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3a8be5..d536935 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Aug 11 2025 Lumír Balhar - 3.14-5 +- import_all_modules: Add error handling for import failures + * Fri Jul 25 2025 Fedora Release Engineering - Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 8b89bc4..dd10689 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['this_is_a_module_in_cwd']) @@ -175,7 +175,7 @@ def test_modules_from_files_are_found(tmp_path): def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,24 +253,70 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_module_with_error_propagates_exception(tmp_path): +def test_import_module_returns_failed_modules(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + assert failed_modules == ['this_is_a_module_in_tmp_path_2'] + + +def test_module_with_error_propagates_exception(tmp_path, capsys): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - # The correct exception must be raised - with pytest.raises(ZeroDivisionError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) + _, err = capsys.readouterr() + assert "ZeroDivisionError" in err + + +def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) + assert failed_modules == [] + + +def test_all_modules_are_imported(tmp_path, capsys): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + with pytest.raises(SystemExit): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'missing_module', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ]) + _, err = capsys.readouterr() + for i in range(1, 4): + assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err + assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): From 7d4cb5437d4f813b20dc2be44fe5d6517f48464f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 22 Jul 2025 20:51:50 +0200 Subject: [PATCH 41/44] Introduce %python_wheel_inject_sbom See https://discuss.python.org/t/encoding-origin-in-wheel-and-dist-info-metadata-for-downstream-security-backports/97436 --- macros.python-wheel-sbom | 124 +++++++++++++++++++++++++++++++++++++++ plan.fmf | 5 ++ python-rpm-macros.spec | 7 ++- tests/testwheel.spec | 102 ++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 macros.python-wheel-sbom create mode 100644 tests/testwheel.spec diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom new file mode 100644 index 0000000..a21460a --- /dev/null +++ b/macros.python-wheel-sbom @@ -0,0 +1,124 @@ +# The macros in this file are used to add SBOM to wheel files that we ship. +# Majority of Python packages will not need to do that, +# as they only use wheels as an intermediate artifact. +# The macros will be used by packages installing wheel to %%python_wheel_dir +# or by Python interpreters bundling their own (patched) wheels. +# +# The runtime dependencies are not Required by the python-rpm-macros package, +# users of this macro need to specify them on their own or rely on the fact that +# they are all available in the default buildroot. +# +# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS +# +# The wheels are modified in-place. + + +# Path of the SBOM file in the PEP 770 .dist-info/sboms directory +# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ +# section Recognized file patterns +%__python_wheel_sbom_filename bom.json + + +# The SBOM content to put to the file +# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 +%__python_wheel_sbom_content %{expand:{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "%{name}", + "version": "%{version}-%{release}", + "purl": "%{__python_wheel_purl}" + } + ] +}} + + +# The purl used above +# We use the src package name (which is easier to get and more useful to consumers). +# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 +# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ +%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src + + +# The purl namespace used above +# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ +# Distributors, define %%dist_purl_namespace to set this. +# The rest of the code is fallback for distributions without it (relying on %%dist_name). +%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: + if macros.epel then + -- being epel beats the %%dist_name value + -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 + print("epel") + else + local dist_map = { + -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm + -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 + ["Fedora Linux"] = "fedora", + -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 + ["CentOS Stream"] = "centos", + -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ + ["Red Hat Enterprise Linux"] = "redhat", + -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html + ["AlmaLinux"] = "almalinux", + -- from https://github.com/google/osv.dev/pull/2939 + ["Rocky Linux"] = "rocky-linux", + } + print(dist_map[macros.dist_name] or "unknown") + end +}} + + +# A Bash scriptlet to inject the SBOM file into the wheel(s) +# The macro takes positional nargs+ with wheel paths +# For each wheel, it +# 1. aborts if the SBOM file is already there (it won't override) +# 2. inserts the SBOM file to .dist-info/sboms +# 3. amends .dist-info/RECORD with the added SBOM file +%python_wheel_inject_sbom() %{expand:( + %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] + + set -eu -o pipefail + export LANG=C.utf-8 + + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + pwd0=$(pwd) + ret=0 + + for whl in %{*}; do + cd "$tmpdir" + if [[ "$whl" != /* ]]; then + whl="$pwd0/$whl" + fi + + record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + distinfo="${record%%/RECORD}" + bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" + + if zipinfo -1 "$whl" | grep -qFx "$bom"; then + echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 + ret=1 + continue + fi + + unzip "$whl" "$record" + mkdir "$distinfo/sboms" + echo '%{__python_wheel_sbom_content}' > "$bom" + checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" + size="$(wc --bytes "$bom" | cut -f1 -d' ')" + echo "$bom,$checksum,$size" >> "$record" + + if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then + touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" + fi + + zip -r "$whl" "$record" "$bom" + rm -rf "$distinfo" + cd "$pwd0" + done + + exit $ret +)} + diff --git a/plan.fmf b/plan.fmf index a681cdf..88850dd 100644 --- a/plan.fmf +++ b/plan.fmf @@ -17,6 +17,9 @@ discover: test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - name: rpmlint_clamp_mtime_on test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: python_wheel_inject_sbom + path: /tests + test: rpmbuild -ba testwheel.spec prepare: - name: Install dependencies @@ -27,6 +30,8 @@ prepare: - python-rpm-macros - python3-rpm-macros - python3-devel + - python3-setuptools + - python3-pip - python3-pytest - python3.6 - dnf diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d536935..86d2865 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -8,6 +8,7 @@ Source101: macros.python Source102: macros.python-srpm Source104: macros.python3 Source105: macros.pybytecompile +Source106: macros.python-wheel-sbom # Lua files Source201: python.lua @@ -55,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -149,6 +150,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/pathfix.py @@ -167,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed Aug 13 2025 Miro Hrončok - 3.14-6 +- Introduce %%python_wheel_inject_sbom + * Mon Aug 11 2025 Lumír Balhar - 3.14-5 - import_all_modules: Add error handling for import failures diff --git a/tests/testwheel.spec b/tests/testwheel.spec new file mode 100644 index 0000000..2e761fc --- /dev/null +++ b/tests/testwheel.spec @@ -0,0 +1,102 @@ +Name: testwheel +Epoch: 42 +Version: 1 +Release: 0%{?dist} +Summary: ... +License: MIT +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools >= 61 +BuildRequires: python3-pip + +%description +This builds and installs a wheel which we can then use as a test for +%%python_wheel_inject_sbom. + + +%prep +cat > pyproject.toml << EOF +[project] +name = "testwheel" +version = "1" + +[build-system] +requires = ["setuptools >= 61"] +build-backend = "setuptools.build_meta" +EOF + + +%build +export PIP_CONFIG_FILE=/dev/null +%{python3} -m pip wheel . --no-build-isolation + +# The macro should happily alter multiple wheels, let's make more +for i in {1..5}; do + mkdir ${i} + cp -a *.whl ${i} +done + +# using relative paths should succeed +%python_wheel_inject_sbom {1..5}/*.whl + +# repetitive use should bail out and fail (SBOM is already there) +%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true + +# each wheel should already have it, all should fail individually as well +for i in {1..5}; do + %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true +done + + +%install +mkdir -p %{buildroot}%{python_wheel_dir} +cp -a *.whl %{buildroot}%{python_wheel_dir} + +# using absolute paths should work +%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl + +# and fail when repeated +%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true + + +%check +%define venvsite venv/lib/python%{python3_version}/site-packages +%{python3} -m venv venv +venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl + +test -f %{venvsite}/testwheel-1.dist-info/RECORD +test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD +# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) +sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' + +# this deliberately uses a different mechanism than the macro +# if you are running this test on a different distro, adjust it +%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} + +PYTHONOPTIMIZE=0 %{python3} -c " +import json +with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: + sbom = json.load(fp) +assert len(sbom['components']) == 1 +assert sbom['components'][0]['type'] == 'library' +assert sbom['components'][0]['name'] == 'testwheel' +assert sbom['components'][0]['version'] == '1-0%{?dist}' +assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' +" + +# replace the installation with the original unaltered wheel +venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl +test -f %{venvsite}/testwheel-1.dist-info/RECORD +# no SBOM +test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true + + +%files +%{python_wheel_dir}/*.whl + + +%changelog +* Wed Aug 13 2025 Miro Hrončok - 42:1-0 +- A static changelog with a date, so we can clamp mtimes From 47de23b3c094fafcace77131f4ce6c233f5d1c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 29 Aug 2025 13:46:23 +0200 Subject: [PATCH 42/44] %python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos In python-setuptools-wheel, the macro was confused by setuptools/_vendor/autocommand-2.2.2.dist-info/RECORD (or other vendored RECORDs) --- macros.python-wheel-sbom | 2 +- python-rpm-macros.spec | 5 ++++- tests/testwheel.spec | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom index a21460a..41389f9 100644 --- a/macros.python-wheel-sbom +++ b/macros.python-wheel-sbom @@ -93,7 +93,7 @@ whl="$pwd0/$whl" fi - record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') distinfo="${record%%/RECORD}" bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 86d2865..ae3d558 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -169,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Aug 29 2025 Miro Hrončok - 3.14-7 +- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos + * Wed Aug 13 2025 Miro Hrončok - 3.14-6 - Introduce %%python_wheel_inject_sbom diff --git a/tests/testwheel.spec b/tests/testwheel.spec index 2e761fc..461d7f3 100644 --- a/tests/testwheel.spec +++ b/tests/testwheel.spec @@ -23,7 +23,18 @@ version = "1" [build-system] requires = ["setuptools >= 61"] build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["testwheel*"] EOF +# create a secondary dist-info folder in the project +# we need to ensure this file is not altered +mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info +touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +echo 'recursive-include testwheel/_vendor *' > MANIFEST.in %build @@ -70,6 +81,9 @@ grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/ # a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true + # this deliberately uses a different mechanism than the macro # if you are running this test on a different distro, adjust it %define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} From 9e5c1461f2346d245a18e3650834d548647dff48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 9 Sep 2025 15:00:10 +0200 Subject: [PATCH 43/44] %python_extras_subpkg: Add -v option to specify the required version(-release) This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package). --- macros.python-srpm | 6 ++++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 7ad62d1..f575868 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -230,7 +230,7 @@ end } -%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' @@ -243,6 +243,7 @@ local value_F = rpm.expand('%{-F}') local value_a = rpm.expand('%{-a}') local value_A = rpm.expand('%{-A}') + local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -265,7 +266,8 @@ if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ae3d558..032c913 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Sep 09 2025 Miro Hrončok - 3.14-8 +- %%python_extras_subpkg: Add -v option to specify the required version(-release) +- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) + * Fri Aug 29 2025 Miro Hrončok - 3.14-7 - %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos diff --git a/tests/test_evals.py b/tests/test_evals.py index b26ff29..31fb75d 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -658,6 +658,21 @@ def test_python_extras_subpkg_aA(): 'BuildArch: noarch (default)) options are not possible') +def test_python_extras_subpkg_v(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 1.2.3 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From 0d417402dbc243c6fa3170fff3e8d6b5b59638c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Oct 2025 16:10:53 +0200 Subject: [PATCH 44/44] %python_extras_subpkg: Only %ghost the egg-info/dist-info directory, not the content That way, accidentally unpackaged files within are reported as errors. Currently, when %python_extras_subpkg is used, the egg-info/dist-info directory is packaged as %ghost. When the main package does not have it, the RPM build would succeed. The extras packages would have the python3dist() requires and provides, but the main package would not. By adding %dir after %ghost, we only package the directory (which is enough for python3-rpm-generators to process it), but the files in the directory are not included. When not packaged in the main package, the RPM build fails. This is a safeguard against packaging mistakes. The visible difference is that rpm -ql/repoquery -l would only return the metadata directory. And the RPM build would fail if .egg-info is a file, which is only possible with Python < 3.12 for packages using distutils (no extras anyway). --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f575868..d5cd4e5 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -287,7 +287,7 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 032c913..263853a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Oct 16 2025 Miro Hrončok - 3.14-9 +- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content +- That way, accidentally unpackaged files within are reported as errors + * Tue Sep 09 2025 Miro Hrončok - 3.14-8 - %%python_extras_subpkg: Add -v option to specify the required version(-release) - This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) diff --git a/tests/test_evals.py b/tests/test_evals.py index 31fb75d..e6cc8f7 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -551,7 +551,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -562,7 +562,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected