From c41028c1376247a335ae232f964eac54d5d03541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 1 Mar 2024 00:41:41 +0100 Subject: [PATCH] brp-fix-pyc-reproducibility: automatically call on whole of /usr Architecture-specific pyc files cause two problems: - noarch packages may cause noarch packages to differ between different architectures in the case where we build an archful package which has some noarch subpackages, - package rebuilds report a difference when build reproducibility is checked and the build happened on a different architecture. Both problems can be resolved by using the existing %py_reproducible_pyc_path macro, but this requires maintainers to opt-in. Let's just extend this operation to all files under /usr, to make life easier for maintainers and for folks working on reproducible builds. "find /usr/ -name '*.pyc'" reveals that .pyc files are installed in quite a few locations, not just the python site and arch directories, so just apply the fixer to the whole /usr. %py_fix_reproducibility can be unset to opt out. %py_reproducible_pyc_path retains its function. We can probably drop all or almost all of its uses in Fedora, but it might be useful for people doing some special things. Fixes https://pagure.io/fedora-reproducible-builds/project/issue/12, https://bugzilla.redhat.com/show_bug.cgi?id=2266767. To deal with bootstrapping, the script does nothing if marshalparser is not installed. python-srpm-macros Require marshalparser, but not when built with bootstrap. The call to marshalparser takes a bit of time. I ran it on all .pyc files on my system and that took a few minutes. But for each single package, the time is usually < 1 s. There are some exceptions, for example libvirt.cpython-312.*.pyc take ~15 s each. But I don't think this really matters: the files come from a huge package which takes a long time to compile, and adding a few dozen seconds at the end doesn't change much. I expect that marshalparser itself might need some optimizations. We can do that at some later point if the slowdowns become noticeable. --- brp-fix-pyc-reproducibility | 17 +++++++++++++---- macros.python-srpm | 6 +++++- python-rpm-macros.spec | 12 +++++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 8047890..95e481b 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -5,14 +5,23 @@ if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then exit 0 fi -# Defined as %py_reproducible_pyc_path macro and passed here as -# the first command-line argument +automatic= +if [ "${1:?}" = "-a" ]; then + automatic=yes + shift +fi + path_to_fix=${1:?} # First, check that the parser is available: if [ ! -x /usr/bin/marshalparser ]; then - echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" - exit 1 + if [ -n "$automatic" ]; then + echo "WARNING: marshalparser is not installed" + exit 0 + else + echo "ERROR: marshalparser is not installed" + exit 1 + fi fi # Set pipefail so if $path_to_fix does not exist, the build fails diff --git a/macros.python-srpm b/macros.python-srpm index 278571f..490f91e 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -77,6 +77,9 @@ ## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] +## Automatically apply __brp_fix_pyc_reproducibility +%py_fix_reproducibility 1 + ## The individual BRP scripts %__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -89,7 +92,8 @@ # 3. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ - %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ + %{?py_fix_reproducibility:%{__brp_fix_pyc_reproducibility} -a "%{buildroot}%{_prefix}"} \ + %{?py_reproducible_pyc_path:%{__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ %{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 06fc622..dcdd307 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -3,6 +3,8 @@ Summary: The common Python RPM macros URL: https://src.fedoraproject.org/rpms/python-rpm-macros/ +%bcond bootstrap 0 + # Macros: Source101: macros.python Source102: macros.python-srpm @@ -53,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -82,6 +84,10 @@ Summary: RPM macros for building Python source packages # For directory structure and flags macros # Versions before 190 contained some brp scripts moved into python-srpm-macros Requires: redhat-rpm-config >= 190 +# Require marshalparser if we're not bootstrapping and python3 is installed +%if %{without bootstrap} +Requires: (marshalparser if python3) +%endif # We bundle our own software here :/ Provides: bundled(python3dist(compileall2)) = %{compileall2_version} @@ -163,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Mar 1 2024 Zbigniew Jedrzejewski-Szmek - 3.12-8 +- Call %%__brp_fix_pyc_reproducibility for all .pyc files under /usr + (rhbz#2266767) + * Thu Jan 25 2024 Miro Hrončok - 3.12-7 - %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set