From 86a7ec8de5a6de252986e63c8268cb394cb493f5 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Thu, 14 Jan 2016 14:32:55 -0700 Subject: [PATCH 001/218] EPEL version --- macros.python-srpm | 7 ++++++- python-rpm-macros.spec | 11 ++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 4257beb..b515582 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -1,3 +1,8 @@ # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) -%python3_pkgversion 3 +%python3_pkgversion 34 +#python3_other_pkgversion 35 +# Set to /bin/true when not active to avoid %ifdefs in specfiles +%__python3_other /bin/true +%py3_other_build /bin/true +%py3_other_install /bin/true diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d71e19c..52a97a2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 4%{?dist} +Release: 4%{?dist}.1 Summary: The unversioned Python RPM macros License: MIT @@ -10,8 +10,6 @@ Source2: macros.python2 Source3: macros.python3 BuildArch: noarch -Obsoletes: python-macros < 3 -Provides: python-macros = %{version}-%{release} %description This package contains the unversioned Python RPM macros, that most @@ -28,16 +26,12 @@ RPM macros for building Python source packages. %package -n python2-rpm-macros Summary: RPM macros for building Python 2 packages -# Would need to be different for each release - worth it? -#Conflicts: python2-devel < 2.7.11-3 %description -n python2-rpm-macros RPM macros for building Python 2 packages. %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages -# Would need to be different for each release - worth it? -#Conflicts: python3-devel < 3.5.1-3 %description -n python3-rpm-macros RPM macros for building Python 3 packages. @@ -67,6 +61,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Thu Jan 14 2016 Orion Poplawski 3-4.1 +- EPEL version + * Wed Jan 13 2016 Orion Poplawski 3-4 - Fix python2/3-rpm-macros package names From 237061d265a11c86275ecae37d4c0117820ed79c Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Thu, 14 Jan 2016 14:46:42 -0700 Subject: [PATCH 002/218] Handle python3_other_pkgversion in python_provide; Drop duplicate py3_other macros from macros.python --- macros.python | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/macros.python b/macros.python index 4a5cb2a..a4c2fc8 100644 --- a/macros.python +++ b/macros.python @@ -29,8 +29,10 @@ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --ski print(string.sub(package,9,string.len(package)));\ print(" < ");\ print(vr);\ - elseif (string.starts(package, "python3-")) then\ + elseif (string.starts(str, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then\ --No unversioned provides as python3 is not default\ + elseif (rpm.expand("%{?python3_other_pkgversion}") != "" and string.starts(str, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then\ + --No unversioned provides as python3_other is not default\ elseif (string.starts(package, "pypy-")) then\ --No unversioned provides as pypy is not default\ elseif (string.starts(package, "pypy3-")) then\ @@ -47,8 +49,3 @@ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --ski print("not recognized.");\ end\ } - -# Set to /bin/true to avoid %ifdefs in specfiles -%__python3_other /bin/true -%py3_other_build /bin/true -%py3_other_install /bin/true From cc0b15603c86a16c2727a7a19bf8b93442ac046d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Mon, 25 Jan 2016 16:55:23 -0700 Subject: [PATCH 003/218] Set %__python3 to /usr/bin/python3.4 --- macros.python3 | 4 +++- python-rpm-macros.spec | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/macros.python3 b/macros.python3 index a633c72..7f00724 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,4 +1,6 @@ -%__python3 /usr/bin/python3 +%__python3 /usr/bin/python3.4 +#__python3_other /usr/bin/python3.5 + %python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3])") diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b703fcb..b720e8d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 6%{?dist} +Release: 6%{?dist}.1 Summary: The unversioned Python RPM macros License: MIT @@ -61,6 +61,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Mon Jan 25 2016 Orion Poplawski 3-6.1 +- Set %%__python3 to /usr/bin/python3.4 + * Thu Jan 14 2016 Orion Poplawski 3-6 - Fix typo in %%python_provide From 59b1647971715dcb8c8fc477987badd379f37552 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 24 Aug 2016 16:47:08 -0600 Subject: [PATCH 004/218] Use %rpmmacrodir --- python-rpm-macros.spec | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d63753d..f0804c1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 10%{?dist} +Release: 11%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -50,25 +50,28 @@ RPM macros for building Python 3 packages. %build %install -mkdir -p %{buildroot}/%{_rpmconfigdir}/macros.d/ +mkdir -p %{buildroot}/%{rpmmacrodir} install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ - %{buildroot}/%{_rpmconfigdir}/macros.d/ + %{buildroot}/%{rpmmacrodir}/ %files -%{_rpmconfigdir}/macros.d/macros.python +%{rpmmacrodir}/macros.python %files -n python-srpm-macros -%{_rpmconfigdir}/macros.d/macros.python-srpm +%{rpmmacrodir}/macros.python-srpm %files -n python2-rpm-macros -%{_rpmconfigdir}/macros.d/macros.python2 +%{rpmmacrodir}/macros.python2 %files -n python3-rpm-macros -%{_rpmconfigdir}/macros.d/macros.python3 +%{rpmmacrodir}/macros.python3 %changelog +* Wed Aug 24 2016 Orion Poplawski 3-11 +- Use %%rpmmacrodir + * Tue Jul 12 2016 Orion Poplawski 3-10 - Do not generate useless Obsoletes with %%{?_isa} From 83441663a4ad710ec9c07d90743944ccee31b684 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Thu, 15 Sep 2016 15:40:12 -0500 Subject: [PATCH 005/218] Fix up issue with python2_version* macros Also move the macros to the proper location (%rpmmacrodir) and copy over one fix from the macros that were in epel-rpm-macros. --- macros.python2 | 6 +++--- python-rpm-macros.spec | 22 +++++++++++++++------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/macros.python2 b/macros.python2 index 789ca09..66e9c5a 100644 --- a/macros.python2 +++ b/macros.python2 @@ -1,8 +1,8 @@ -%__python2 /usr/bin/python2 +%__python2 /usr/bin/python2.6 %python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python2_sitearch %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python2_version %(%{__python2} -c "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python2_version_nodots %(%{__python2} -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python2_version %(%{__python2} -c "import sys; sys.stdout.write(sys.version[:3])") +%python2_version_nodots %(%{__python2} -c "import sys; sys.stdout.write(sys.version[:3:2])") %py2_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 51b5e40..f2c2bff 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 10%{?dist} +Release: 11%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -44,25 +44,33 @@ RPM macros for building Python 3 packages. %build %install -mkdir -p %{buildroot}/%{_rpmconfigdir}/macros.d/ +mkdir -p %{buildroot}/%{rpmmacrodir} install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ - %{buildroot}/%{_rpmconfigdir}/macros.d/ + %{buildroot}/%{rpmmacrodir} %files -%{_rpmconfigdir}/macros.d/macros.python +%{rpmmacrodir}/macros.python %files -n python-srpm-macros -%{_rpmconfigdir}/macros.d/macros.python-srpm +%{rpmmacrodir}/macros.python-srpm %files -n python2-rpm-macros -%{_rpmconfigdir}/macros.d/macros.python2 +%{rpmmacrodir}/macros.python2 %files -n python3-rpm-macros -%{_rpmconfigdir}/macros.d/macros.python3 +%{rpmmacrodir}/macros.python3 %changelog +* Thu Sep 15 2016 Jason L Tibbitts III - 3-11 +- Use %%rpmmacrodir to put the macros in the proper location. +- Change %%python2_version* to a form that works on EL6. +- Change %%__python2 to point explicitly to python2.6 to avoid breakage in + Amazon Linux, which links /usr/bin/python to python2.7. This syncs up with + the macros which were in epel-rpm-macros. See + https://bugzilla.redhat.com/show_bug.cgi?id=1347019 + * Tue Jul 12 2016 Orion Poplawski 3-10 - Do not generate useless Obsoletes with %%{?_isa} From 29e6636ccda30e3d6b64ab832e34648993a94e2d Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Tue, 15 Nov 2016 16:16:16 -0700 Subject: [PATCH 006/218] Tidy up macro formatting --- macros.python | 86 +++++++++++++++++++++++++------------------------- macros.python2 | 10 +++--- macros.python3 | 10 +++--- 3 files changed, 53 insertions(+), 53 deletions(-) diff --git a/macros.python b/macros.python index 94e769c..12b626c 100644 --- a/macros.python +++ b/macros.python @@ -1,51 +1,51 @@ %py_setup setup.py %py_shbang_opts -s -%py_build() %{expand:\ -CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?1}\ +%py_build() %{expand: + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?1} } -%py_install() %{expand:\ -CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1}\ +%py_install() %{expand: + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} } -%python_provide() %{lua:\ - function string.starts(String,Start)\ - return string.sub(String,1,string.len(Start))==Start\ - end\ - package = rpm.expand("%{?1:%{1}}");\ - vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") - if (string.starts(package, "python2-")) then\ - if (rpm.expand("%{?buildarch}") ~= "noarch") then\ - str = "Provides: python-" .. string.sub(package,9,string.len(package)) .. "%{?_isa} = " .. vr;\ - print(rpm.expand(str));\ - end\ - print("\\nProvides: python-");\ - print(string.sub(package,9,string.len(package)));\ - print(" = ");\ - print(vr);\ - --Obsoleting the previous default python package\ - print("\\nObsoletes: python-");\ - print(string.sub(package,9,string.len(package)));\ - print(" < ");\ - print(vr);\ - elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then\ - --No unversioned provides as python3 is not default\ - elseif (rpm.expand("%{?python3_other_pkgversion}") ~= "" and string.starts(package, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then\ - --No unversioned provides as python3_other is not default\ - elseif (string.starts(package, "pypy-")) then\ - --No unversioned provides as pypy is not default\ - elseif (string.starts(package, "pypy3-")) then\ - --No unversioned provides as pypy is not default\ - elseif (string.starts(package, "python-")) then\ - --Providing the current default python\ - print("Provides: python2-");\ - print(string.sub(package,8,string.len(package)));\ - print(" = ");\ - print(vr);\ - else\ - print("%python_provide: ERROR: ");\ - print(package);\ - print(" not recognized.");\ - end\ +%python_provide() %{lua: + function string.starts(String,Start) + return string.sub(String,1,string.len(Start))==Start + end + package = rpm.expand("%{?1:%{1}}") + vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") + if (string.starts(package, "python2-")) then + if (rpm.expand("%{?buildarch}") ~= "noarch") then + str = "Provides: python-" .. string.sub(package,9,string.len(package)) .. "%{?_isa} = " .. vr + print(rpm.expand(str)) + end + print("\\nProvides: python-") + print(string.sub(package,9,string.len(package))) + print(" = ") + print(vr) + --Obsoleting the previous default python package + print("\\nObsoletes: python-") + print(string.sub(package,9,string.len(package))) + print(" < ") + print(vr) + elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then + --No unversioned provides as python3 is not default + elseif (rpm.expand("%{?python3_other_pkgversion}") ~= "" and string.starts(package, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then + --No unversioned provides as python3_other is not default + elseif (string.starts(package, "pypy-")) then + --No unversioned provides as pypy is not default + elseif (string.starts(package, "pypy3-")) then + --No unversioned provides as pypy is not default + elseif (string.starts(package, "python-")) then + --Providing the current default python + print("Provides: python2-") + print(string.sub(package,8,string.len(package))) + print(" = ") + print(vr) + else + print("%python_provide: ERROR: ") + print(package) + print(" not recognized.") + end } diff --git a/macros.python2 b/macros.python2 index 789ca09..b39a1b1 100644 --- a/macros.python2 +++ b/macros.python2 @@ -6,11 +6,11 @@ %py2_shbang_opts -s -%py2_build() %{expand:\ -CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?1};\ -sleep 1\ +%py2_build() %{expand: + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?1} + sleep 1 } -%py2_install() %{expand:\ -CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1}\ +%py2_install() %{expand: + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} } diff --git a/macros.python3 b/macros.python3 index 4f44bd1..f083b7a 100644 --- a/macros.python3 +++ b/macros.python3 @@ -7,11 +7,11 @@ %py3_shbang_opts -s -%py3_build() %{expand:\ -CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?1};\ -sleep 1\ +%py3_build() %{expand: + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?1} + sleep 1 } -%py3_install() %{expand:\ -CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1}\ +%py3_install() %{expand: + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} } From 6ce524f21d4b087f6cff6f98a0ee6e1eb404eff7 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Tue, 15 Nov 2016 16:18:39 -0700 Subject: [PATCH 007/218] Allow multiple args to %py_build/install macros --- macros.python | 6 +++--- macros.python2 | 4 ++-- macros.python3 | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/macros.python b/macros.python index 12b626c..2913a27 100644 --- a/macros.python +++ b/macros.python @@ -2,18 +2,18 @@ %py_shbang_opts -s %py_build() %{expand: - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?1} + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} } %py_install() %{expand: - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } %python_provide() %{lua: function string.starts(String,Start) return string.sub(String,1,string.len(Start))==Start end - package = rpm.expand("%{?1:%{1}}") + package = rpm.expand("%{?1}") vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") if (string.starts(package, "python2-")) then if (rpm.expand("%{?buildarch}") ~= "noarch") then diff --git a/macros.python2 b/macros.python2 index b39a1b1..a79b941 100644 --- a/macros.python2 +++ b/macros.python2 @@ -7,10 +7,10 @@ %py2_shbang_opts -s %py2_build() %{expand: - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?1} + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} sleep 1 } %py2_install() %{expand: - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } diff --git a/macros.python3 b/macros.python3 index f083b7a..665f9f2 100644 --- a/macros.python3 +++ b/macros.python3 @@ -8,10 +8,10 @@ %py3_shbang_opts -s %py3_build() %{expand: - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?1} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} sleep 1 } %py3_install() %{expand: - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?1} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } From f798c4335426a9c900a42ed28f34402416b0f4f8 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Tue, 15 Nov 2016 16:12:23 -0700 Subject: [PATCH 008/218] Add %py_build_egg and %py_install_egg macros --- macros.python | 9 +++++++++ macros.python2 | 10 ++++++++++ macros.python3 | 10 ++++++++++ 3 files changed, 29 insertions(+) diff --git a/macros.python b/macros.python index 2913a27..710678e 100644 --- a/macros.python +++ b/macros.python @@ -5,10 +5,19 @@ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} } +%py_build_egg() %{expand: + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python2} %{py_shbang_opts}" %{?*} +} + %py_install() %{expand: CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } +%py_install_egg() %{expand: + mkdir -p %{buildroot}%{python_sitelib} + easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} +} + %python_provide() %{lua: function string.starts(String,Start) return string.sub(String,1,string.len(Start))==Start diff --git a/macros.python2 b/macros.python2 index a79b941..b5bc98a 100644 --- a/macros.python2 +++ b/macros.python2 @@ -11,6 +11,16 @@ sleep 1 } +%py2_build_egg() %{expand: + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python2} %{py2_shbang_opts}" %{?*} + sleep 1 +} + %py2_install() %{expand: CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } + +%py2_install_egg() %{expand: + mkdir -p %{buildroot}%{python2_sitelib} + easy_install-%{python2_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} +} diff --git a/macros.python3 b/macros.python3 index 665f9f2..fcf9964 100644 --- a/macros.python3 +++ b/macros.python3 @@ -12,6 +12,16 @@ sleep 1 } +%py3_build_egg() %{expand: + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python3} %{py3_shbang_opts}" %{?*} + sleep 1 +} + %py3_install() %{expand: CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } + +%py3_install_egg() %{expand: + mkdir -p %{buildroot}%{python3_sitelib} + easy_install-%{python3_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} +} From bad56e0efe80f4e0d851bd83e77902a1f2f70a9f Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Tue, 15 Nov 2016 16:26:40 -0700 Subject: [PATCH 009/218] Bump release and add %changelog entry --- python-rpm-macros.spec | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index f0804c1..a1cfffa 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 11%{?dist} +Release: 12%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Tue Nov 15 2016 Orion Poplawski 3-12 +- Add %%py_build_egg and %%py_install_egg macros +- Allow multiple args to %%py_build/install macros +- Tidy up macro formatting + * Wed Aug 24 2016 Orion Poplawski 3-11 - Use %%rpmmacrodir From e7af332922fbd543df3844d56313cd1f8b2d336c Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 16 Nov 2016 16:28:31 -0700 Subject: [PATCH 010/218] Add missing sleeps to other build macros - Fix build_egg macros - Add %py_build_wheel and %py_install_wheel macros --- macros.python | 13 ++++++++++++- macros.python2 | 11 ++++++++++- macros.python3 | 10 +++++++++- python-rpm-macros.spec | 7 ++++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/macros.python b/macros.python index 710678e..a0311ca 100644 --- a/macros.python +++ b/macros.python @@ -3,10 +3,17 @@ %py_build() %{expand: CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} + sleep 1 } %py_build_egg() %{expand: - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python2} %{py_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + sleep 1 +} + +%py_build_wheel() %{expand: + CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} + sleep 1 } %py_install() %{expand: @@ -18,6 +25,10 @@ easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} } +%py_install_wheel() %{expand: + pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} +} + %python_provide() %{lua: function string.starts(String,Start) return string.sub(String,1,string.len(Start))==Start diff --git a/macros.python2 b/macros.python2 index b5bc98a..81e3aa5 100644 --- a/macros.python2 +++ b/macros.python2 @@ -12,7 +12,12 @@ } %py2_build_egg() %{expand: - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python2} %{py2_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + sleep 1 +} + +%py2_build_wheel() %{expand: + CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } @@ -24,3 +29,7 @@ mkdir -p %{buildroot}%{python2_sitelib} easy_install-%{python2_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} } + +%py2_install_wheel() %{expand: + pip%{python2_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} +} diff --git a/macros.python3 b/macros.python3 index fcf9964..984dbcd 100644 --- a/macros.python3 +++ b/macros.python3 @@ -13,7 +13,12 @@ } %py3_build_egg() %{expand: - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_egg --executable="%{__python3} %{py3_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + sleep 1 +} + +%py3_build_wheel() %{expand: + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } @@ -25,3 +30,6 @@ mkdir -p %{buildroot}%{python3_sitelib} easy_install-%{python3_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} } + +%py3_install_wheel() %{expand: + pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a1cfffa..62c7977 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 12%{?dist} +Release: 13%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Wed Nov 16 2016 Orion Poplawski 3-13 +- Add missing sleeps to other build macros +- Fix build_egg macros +- Add %%py_build_wheel and %%py_install_wheel macros + * Tue Nov 15 2016 Orion Poplawski 3-12 - Add %%py_build_egg and %%py_install_egg macros - Allow multiple args to %%py_build/install macros From a73eb24716ec8777f857f2419de2bfcf05eb40e3 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Wed, 16 Nov 2016 21:32:46 -0700 Subject: [PATCH 011/218] Fix %py3_install_wheel (bug #1395953) --- macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index 984dbcd..2cabc40 100644 --- a/macros.python3 +++ b/macros.python3 @@ -33,3 +33,4 @@ %py3_install_wheel() %{expand: pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 62c7977..ed94d9d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 13%{?dist} +Release: 14%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Wed Nov 16 2016 Orion Poplawski 3-14 +- Fix %%py3_install_wheel (bug #1395953) + * Wed Nov 16 2016 Orion Poplawski 3-13 - Add missing sleeps to other build macros - Fix build_egg macros From 5ca1f525e5551e4c80d4168e5eebd56c95eda531 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Thu, 24 Nov 2016 07:53:47 -0700 Subject: [PATCH 012/218] Make expaned macros start on the same line as the macro --- macros.python | 14 ++++++++------ macros.python2 | 14 ++++++++------ macros.python3 | 14 ++++++++------ python-rpm-macros.spec | 5 ++++- 4 files changed, 28 insertions(+), 19 deletions(-) diff --git a/macros.python b/macros.python index a0311ca..aab13c0 100644 --- a/macros.python +++ b/macros.python @@ -1,31 +1,33 @@ %py_setup setup.py %py_shbang_opts -s -%py_build() %{expand: +# Use the slashes after expand so that the command starts on the same line as +# the macro +%py_build() %{expand:\\\ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} sleep 1 } -%py_build_egg() %{expand: +%py_build_egg() %{expand:\\\ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } -%py_build_wheel() %{expand: +%py_build_wheel() %{expand:\\\ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } -%py_install() %{expand: +%py_install() %{expand:\\\ CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } -%py_install_egg() %{expand: +%py_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python_sitelib} easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} } -%py_install_wheel() %{expand: +%py_install_wheel() %{expand:\\\ pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} } diff --git a/macros.python2 b/macros.python2 index 81e3aa5..420544b 100644 --- a/macros.python2 +++ b/macros.python2 @@ -6,30 +6,32 @@ %py2_shbang_opts -s -%py2_build() %{expand: +# Use the slashes after expand so that the command starts on the same line as +# the macro +%py2_build() %{expand:\\\ CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} sleep 1 } -%py2_build_egg() %{expand: +%py2_build_egg() %{expand:\\\ CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } -%py2_build_wheel() %{expand: +%py2_build_wheel() %{expand:\\\ CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } -%py2_install() %{expand: +%py2_install() %{expand:\\\ CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } -%py2_install_egg() %{expand: +%py2_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python2_sitelib} easy_install-%{python2_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} } -%py2_install_wheel() %{expand: +%py2_install_wheel() %{expand:\\\ pip%{python2_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} } diff --git a/macros.python3 b/macros.python3 index 2cabc40..c81cf3f 100644 --- a/macros.python3 +++ b/macros.python3 @@ -7,30 +7,32 @@ %py3_shbang_opts -s -%py3_build() %{expand: +# Use the slashes after expand so that the command starts on the same line as +# the macro +%py3_build() %{expand:\\\ CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} sleep 1 } -%py3_build_egg() %{expand: +%py3_build_egg() %{expand:\\\ CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } -%py3_build_wheel() %{expand: +%py3_build_wheel() %{expand:\\\ CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } -%py3_install() %{expand: +%py3_install() %{expand:\\\ CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } -%py3_install_egg() %{expand: +%py3_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python3_sitelib} easy_install-%{python3_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} } -%py3_install_wheel() %{expand: +%py3_install_wheel() %{expand:\\\ pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ed94d9d..055a3f0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 14%{?dist} +Release: 15%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Thu Nov 24 2016 Orion Poplawski 3-15 +- Make expaned macros start on the same line as the macro + * Wed Nov 16 2016 Orion Poplawski 3-14 - Fix %%py3_install_wheel (bug #1395953) From 516c55eed718753e8484a1c45bb87eb8236e8282 Mon Sep 17 00:00:00 2001 From: Orion Poplawski Date: Thu, 24 Nov 2016 07:59:27 -0700 Subject: [PATCH 013/218] Fix typo in changelog --- python-rpm-macros.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 055a3f0..1ab3223 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -70,7 +70,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog * Thu Nov 24 2016 Orion Poplawski 3-15 -- Make expaned macros start on the same line as the macro +- Make expanded macros start on the same line as the macro * Wed Nov 16 2016 Orion Poplawski 3-14 - Fix %%py3_install_wheel (bug #1395953) From 123ad4b49f392a6fb61c8a23caab44fc8ae163b5 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 17 Jan 2017 10:28:13 +0100 Subject: [PATCH 014/218] Added macros for Build/Requires tags using Python dist tags https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages --- macros.python-srpm | 55 ++++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 6 ++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 87ce887..38a71dc 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -6,3 +6,58 @@ %__python3_other /bin/true %py3_other_build /bin/true %py3_other_install /bin/true + + + +# === Macros for Build/Requires tags using Python dist tags === +# - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages +# - These macros need to be in macros.python-srpm, because BuildRequires tags +# get rendered as runtime requires into the metadata of SRPMs. + +# Converts Python dist name to a canonical format +%py_dist_name() %{lua:\ + name = rpm.expand("%{?1:%{1}}");\ + canonical = string.gsub(string.lower(name), "%W+", "-");\ + print(canonical);\ +} + +# Creates Python 2 dist tag(s) after converting names to canonical format +# Needs to first put all arguments into a list, because invoking a different +# macro (%py_dist_name) overwrites them +%py2_dist() %{lua:\ + args = {}\ + arg = 1\ + while (true) do\ + name = rpm.expand("%{?" .. arg .. ":%{" .. arg .. "}}");\ + if (name == nil or name == '') then\ + break\ + end\ + args[arg] = name\ + arg = arg + 1\ + end\ + for arg, name in ipairs(args) do\ + canonical = rpm.expand("%py_dist_name " .. name);\ + print("python2dist(" .. canonical .. ") ");\ + end\ +} + +# Creates Python 3 dist tag(s) after converting names to canonical format +# Needs to first put all arguments into a list, because invoking a different +# macro (%py_dist_name) overwrites them +%py3_dist() %{lua:\ + args = {}\ + arg = 1\ + while (true) do\ + name = rpm.expand("%{?" .. arg .. ":%{" .. arg .. "}}");\ + if (name == nil or name == '') then\ + break\ + end\ + args[arg] = name\ + arg = arg + 1\ + end\ + for arg, name in ipairs(args) do\ + canonical = rpm.expand("%py_dist_name " .. name);\ + print("python3dist(" .. canonical .. ") ");\ + end\ +} + diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 1ab3223..32d1604 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 15%{?dist} +Release: 16%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Tue Jan 17 2017 Tomas Orsava - 3-16 +- Added macros for Build/Requires tags using Python dist tags: + https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages + * Thu Nov 24 2016 Orion Poplawski 3-15 - Make expanded macros start on the same line as the macro From d0fdb0d759b960b41b57bf7c9539901f16062e0a Mon Sep 17 00:00:00 2001 From: Michal Cyprian Date: Mon, 23 Jan 2017 18:52:17 +0100 Subject: [PATCH 015/218] Add --no-deps option to py_install_wheel macros --- macros.python | 2 +- macros.python2 | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/macros.python b/macros.python index aab13c0..8b168b5 100644 --- a/macros.python +++ b/macros.python @@ -28,7 +28,7 @@ } %py_install_wheel() %{expand:\\\ - pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} + pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps } %python_provide() %{lua: diff --git a/macros.python2 b/macros.python2 index 420544b..169d48f 100644 --- a/macros.python2 +++ b/macros.python2 @@ -33,5 +33,5 @@ } %py2_install_wheel() %{expand:\\\ - pip%{python2_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} + pip%{python2_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps } diff --git a/macros.python3 b/macros.python3 index c81cf3f..2805b8a 100644 --- a/macros.python3 +++ b/macros.python3 @@ -34,5 +34,5 @@ } %py3_install_wheel() %{expand:\\\ - pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} + pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 32d1604..be402c9 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 16%{?dist} +Release: 17%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Mon Jan 23 2017 Michal Cyprian - 3-17 +- Add --no-deps option to py_install_wheel macros + * Tue Jan 17 2017 Tomas Orsava - 3-16 - Added macros for Build/Requires tags using Python dist tags: https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages From 063c3984a93c69545ef5185708a7b11406b3d003 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 11 Feb 2017 10:23:28 +0000 Subject: [PATCH 016/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index be402c9..0e45ceb 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 17%{?dist} +Release: 18%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Sat Feb 11 2017 Fedora Release Engineering - 3-18 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild + * Mon Jan 23 2017 Michal Cyprian - 3-17 - Add --no-deps option to py_install_wheel macros From d019d2fd4d07081a598b9d3e45d839665adab85e Mon Sep 17 00:00:00 2001 From: Michal Cyprian Date: Fri, 17 Feb 2017 14:48:08 +0100 Subject: [PATCH 017/218] Switch %%__python3 to /usr/libexec/system-python --- macros.python3 | 9 +++++---- python-rpm-macros.spec | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/macros.python3 b/macros.python3 index 2805b8a..500af18 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,4 +1,5 @@ -%__python3 /usr/bin/python3 +%__python3 /usr/libexec/system-python +%__python3_bin /usr/bin/python3 %python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3])") @@ -10,7 +11,7 @@ # Use the slashes after expand so that the command starts on the same line as # the macro %py3_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3_bin} %{py3_shbang_opts}" %{?*} sleep 1 } @@ -25,7 +26,7 @@ } %py3_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --executable="%{__python3_bin} %{py3_shbang_opts}" %{?*} } %py3_install_egg() %{expand:\\\ @@ -34,5 +35,5 @@ } %py3_install_wheel() %{expand:\\\ - pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps + pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --prefix %{_prefix} --no-deps } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0e45ceb..65bef6b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 18%{?dist} +Release: 19%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Feb 17 2017 Michal Cyprian - 3-19 +- Switch %%__python3 to /usr/libexec/system-python + * Sat Feb 11 2017 Fedora Release Engineering - 3-18 - Rebuilt for https://fedoraproject.org/wiki/Fedora_26_Mass_Rebuild From a59818afd063ffaafc724b94f9e6a6a8a6530831 Mon Sep 17 00:00:00 2001 From: Michal Cyprian Date: Fri, 3 Mar 2017 13:52:35 +0100 Subject: [PATCH 018/218] Revert "Switch %%__python3 to /usr/libexec/system-python" - The Fedora Change https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe was postponed - This reverts commit d019d2fd4d07081a598b9d3e45d839665adab85e --- macros.python3 | 9 ++++----- python-rpm-macros.spec | 7 ++++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/macros.python3 b/macros.python3 index 500af18..2805b8a 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,5 +1,4 @@ -%__python3 /usr/libexec/system-python -%__python3_bin /usr/bin/python3 +%__python3 /usr/bin/python3 %python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3])") @@ -11,7 +10,7 @@ # Use the slashes after expand so that the command starts on the same line as # the macro %py3_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3_bin} %{py3_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} sleep 1 } @@ -26,7 +25,7 @@ } %py3_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --executable="%{__python3_bin} %{py3_shbang_opts}" %{?*} + CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } %py3_install_egg() %{expand:\\\ @@ -35,5 +34,5 @@ } %py3_install_wheel() %{expand:\\\ - pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --prefix %{_prefix} --no-deps + pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 65bef6b..ac986fa 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 19%{?dist} +Release: 20%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Mar 03 2017 Michal Cyprian - 3-20 +- Revert "Switch %%__python3 to /usr/libexec/system-python" + after the Fedora Change https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe + was postponed + * Fri Feb 17 2017 Michal Cyprian - 3-19 - Switch %%__python3 to /usr/libexec/system-python From b58450b1f3468ec88cb81f7e5599159306b3121a Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Thu, 27 Jul 2017 11:44:18 +0000 Subject: [PATCH 019/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ac986fa..6c0ea74 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 20%{?dist} +Release: 21%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Thu Jul 27 2017 Fedora Release Engineering - 3-21 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild + * Fri Mar 03 2017 Michal Cyprian - 3-20 - Revert "Switch %%__python3 to /usr/libexec/system-python" after the Fedora Change https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe From cb52c18aa91a05410adb3a329d0781001ec8e9b3 Mon Sep 17 00:00:00 2001 From: Iryna Shcherbina Date: Mon, 14 Aug 2017 14:35:46 +0200 Subject: [PATCH 020/218] Add platform-python macros --- macros.platform-python | 38 ++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 17 +++++++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 macros.platform-python diff --git a/macros.platform-python b/macros.platform-python new file mode 100644 index 0000000..97eae6e --- /dev/null +++ b/macros.platform-python @@ -0,0 +1,38 @@ +%__platform_python /usr/libexec/platform-python +%platform_python_sitelib %(%{__platform_python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +%platform_python_sitearch %(%{__platform_python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%platform_python_version %(%{__platform_python} -c "import sys; sys.stdout.write(sys.version[:3])") +%platform_python_version_nodots %(%{__platform_python} -c "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") +%platform_py_dir %{_builddir}/platform-python-%{name}-%{version}-%{release} + +%platform_py_shbang_opts -s + +# Use the slashes after expand so that the command starts on the same line as +# the macro +%platform_py_build() %{expand:\\\ + CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} build --executable="%{__platform_python} %{platform_py_shbang_opts}" %{?*} + sleep 1 +} + +%platform_py_build_egg() %{expand:\\\ + CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + sleep 1 +} + +%platform_py_build_wheel() %{expand:\\\ + CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} + sleep 1 +} + +%platform_py_install() %{expand:\\\ + CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} +} + +%platform_py_install_egg() %{expand:\\\ + mkdir -p %{buildroot}%{platform_python_sitelib} + CFLAGS="%{optflags}" %{__platform_python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{platform_python_version}.egg %{?*} +} + +%platform_py_install_wheel() %{expand:\\\ + CFLAGS="%{optflags}" %{__platform_python} -m pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 6c0ea74..ea14756 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 21%{?dist} +Release: 22%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -8,6 +8,7 @@ Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 +Source4: macros.platform-python BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -44,6 +45,12 @@ Summary: RPM macros for building Python 3 packages %description -n python3-rpm-macros RPM macros for building Python 3 packages. +%package -n platform-python-rpm-macros +Summary: RPM macros for building Platform-Python packages + +%description -n platform-python-rpm-macros +RPM macros for building Platform-Python packages. + %prep @@ -51,7 +58,7 @@ RPM macros for building Python 3 packages. %install mkdir -p %{buildroot}/%{rpmmacrodir} -install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ +install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %{buildroot}/%{rpmmacrodir}/ @@ -67,8 +74,14 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %files -n python3-rpm-macros %{rpmmacrodir}/macros.python3 +%files -n platform-python-rpm-macros +%{rpmmacrodir}/macros.platform-python + %changelog +* Wed Aug 02 2017 Tomas Orsava - 3-22 +- Add platform-python macros (https://fedoraproject.org/wiki/Changes/Platform_Python_Stack) + * Thu Jul 27 2017 Fedora Release Engineering - 3-21 - Rebuilt for https://fedoraproject.org/wiki/Fedora_27_Mass_Rebuild From 8d3d1dde648226944323560698603f5e88afa124 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Wed, 25 Oct 2017 21:56:17 +0300 Subject: [PATCH 021/218] macros.python*: Use -Es/-I to invoke macro scriptlets To avoid environment and user dir influence. https://bugzilla.redhat.com/show_bug.cgi?id=1506355 --- macros.python2 | 8 ++++---- macros.python3 | 8 ++++---- python-rpm-macros.spec | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/macros.python2 b/macros.python2 index 169d48f..fc514c2 100644 --- a/macros.python2 +++ b/macros.python2 @@ -1,8 +1,8 @@ %__python2 /usr/bin/python2 -%python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python2_sitearch %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python2_version %(%{__python2} -c "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python2_version_nodots %(%{__python2} -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python2_sitelib %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +%python2_sitearch %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%python2_version %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %py2_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index 2805b8a..dc15710 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,8 +1,8 @@ %__python3 /usr/bin/python3 -%python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python3_version %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3])") -%python3_version_nodots %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") +%python3_sitelib %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +%python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3])") +%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ea14756..dcd121b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 22%{?dist} +Release: 23%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -79,6 +79,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Thu Oct 26 2017 Ville Skyttä - 3-23 +- Use -Es/-I to invoke macro scriptlets (#1506355) + * Wed Aug 02 2017 Tomas Orsava - 3-22 - Add platform-python macros (https://fedoraproject.org/wiki/Changes/Platform_Python_Stack) From 69b720ea4f67af6d220abfa08e7f9d40f0080d03 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 28 Nov 2017 11:51:13 +0100 Subject: [PATCH 022/218] Revert "Add platform-python macros" This reverts commit cb52c18aa91a05410adb3a329d0781001ec8e9b3. --- macros.platform-python | 38 -------------------------------------- python-rpm-macros.spec | 17 +++++------------ 2 files changed, 5 insertions(+), 50 deletions(-) delete mode 100644 macros.platform-python diff --git a/macros.platform-python b/macros.platform-python deleted file mode 100644 index 97eae6e..0000000 --- a/macros.platform-python +++ /dev/null @@ -1,38 +0,0 @@ -%__platform_python /usr/libexec/platform-python -%platform_python_sitelib %(%{__platform_python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%platform_python_sitearch %(%{__platform_python} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%platform_python_version %(%{__platform_python} -c "import sys; sys.stdout.write(sys.version[:3])") -%platform_python_version_nodots %(%{__platform_python} -c "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") -%platform_py_dir %{_builddir}/platform-python-%{name}-%{version}-%{release} - -%platform_py_shbang_opts -s - -# Use the slashes after expand so that the command starts on the same line as -# the macro -%platform_py_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} build --executable="%{__platform_python} %{platform_py_shbang_opts}" %{?*} - sleep 1 -} - -%platform_py_build_egg() %{expand:\\\ - CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} - sleep 1 -} - -%platform_py_build_wheel() %{expand:\\\ - CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} - sleep 1 -} - -%platform_py_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__platform_python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} -} - -%platform_py_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{platform_python_sitelib} - CFLAGS="%{optflags}" %{__platform_python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{platform_python_version}.egg %{?*} -} - -%platform_py_install_wheel() %{expand:\\\ - CFLAGS="%{optflags}" %{__platform_python} -m pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps -} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index dcd121b..53a74d1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 23%{?dist} +Release: 24%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -8,7 +8,6 @@ Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 -Source4: macros.platform-python BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -45,12 +44,6 @@ Summary: RPM macros for building Python 3 packages %description -n python3-rpm-macros RPM macros for building Python 3 packages. -%package -n platform-python-rpm-macros -Summary: RPM macros for building Platform-Python packages - -%description -n platform-python-rpm-macros -RPM macros for building Platform-Python packages. - %prep @@ -58,7 +51,7 @@ RPM macros for building Platform-Python packages. %install mkdir -p %{buildroot}/%{rpmmacrodir} -install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ +install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %{buildroot}/%{rpmmacrodir}/ @@ -74,11 +67,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %files -n python3-rpm-macros %{rpmmacrodir}/macros.python3 -%files -n platform-python-rpm-macros -%{rpmmacrodir}/macros.platform-python - %changelog +* Tue Nov 28 2017 Tomas Orsava - 3-24 +- Remove platform-python macros (https://fedoraproject.org/wiki/Changes/Platform_Python_Stack) + * Thu Oct 26 2017 Ville Skyttä - 3-23 - Use -Es/-I to invoke macro scriptlets (#1506355) From ec476c84cc342ef1a1b6bb5a87134a3f50fb2930 Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Fri, 19 Jan 2018 15:37:47 +0100 Subject: [PATCH 023/218] add macros to enable dependency generator References: https://fedoraproject.org/wiki/Changes/EnablingPythonGenerators Signed-off-by: Igor Gnatenko --- macros.python | 4 ++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 8b168b5..7c0f058 100644 --- a/macros.python +++ b/macros.python @@ -71,3 +71,7 @@ print(" not recognized.") end } + +%python_enable_dependency_generator() \ +%global __python_requires %{_rpmconfigdir}/pythondistdeps.py --requires \ +%{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 53a74d1..3538e49 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 24%{?dist} +Release: 25%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Jan 19 2018 Igor Gnatenko - 3-25 +- Add %%python_enable_dependency_generator + * Tue Nov 28 2017 Tomas Orsava - 3-24 - Remove platform-python macros (https://fedoraproject.org/wiki/Changes/Platform_Python_Stack) From 9f912209e5009a70056cdf9d902038a16a595e74 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 9 Feb 2018 10:39:49 +0000 Subject: [PATCH 024/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3538e49..e7868bf 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 25%{?dist} +Release: 26%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Feb 09 2018 Fedora Release Engineering - 3-26 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild + * Fri Jan 19 2018 Igor Gnatenko - 3-25 - Add %%python_enable_dependency_generator From 4904408b2f97c614e1ba642e88524dde5e69e4f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Pokorn=C3=BD?= Date: Sat, 3 Feb 2018 12:57:13 +0100 Subject: [PATCH 025/218] macros.python*: make LDFLAGS propagated whenever CFLAGS are MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is to ensure the right linker flags get propagated into binaries accompanying python packages whenever distutils (or similar, env. variables provided build flags aware) module is delegated to build them (e.g. https://bugzilla.redhat.com/show_bug.cgi?id=1541106). Instead of using "%{__global_ldflags}" as the seed for the respective env. variable one-off application, which would be a direct parallel to "%{optflags}" seeding CFLAGS, use rather "${RPM_LD_FLAGS}" and "${RPM_OPT_FLAGS}", as these are not directly bound to macros out of rpm proper (like it would have been, e.g., on redhat-rpm-config package otherwise). Signed-off-by: Jan Pokorný --- macros.python | 12 ++++++++---- macros.python2 | 12 ++++++++---- macros.python3 | 12 ++++++++---- python-rpm-macros.spec | 5 ++++- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/macros.python b/macros.python index 7c0f058..f0039c8 100644 --- a/macros.python +++ b/macros.python @@ -4,22 +4,26 @@ # Use the slashes after expand so that the command starts on the same line as # the macro %py_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} sleep 1 } %py_build_egg() %{expand:\\\ - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } %py_build_wheel() %{expand:\\\ - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } %py_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } %py_install_egg() %{expand:\\\ diff --git a/macros.python2 b/macros.python2 index fc514c2..d6c74c2 100644 --- a/macros.python2 +++ b/macros.python2 @@ -9,22 +9,26 @@ # Use the slashes after expand so that the command starts on the same line as # the macro %py2_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} sleep 1 } %py2_build_egg() %{expand:\\\ - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } %py2_build_wheel() %{expand:\\\ - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } %py2_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } %py2_install_egg() %{expand:\\\ diff --git a/macros.python3 b/macros.python3 index dc15710..79c33f1 100644 --- a/macros.python3 +++ b/macros.python3 @@ -10,22 +10,26 @@ # Use the slashes after expand so that the command starts on the same line as # the macro %py3_build() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} sleep 1 } %py3_build_egg() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } %py3_build_wheel() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 } %py3_install() %{expand:\\\ - CFLAGS="%{optflags}" %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} } %py3_install_egg() %{expand:\\\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e7868bf..8c095d9 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 26%{?dist} +Release: 27%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Mar 23 2018 Miro Hrončok - 3-27 +- make LDFLAGS propagated whenever CFLAGS are + * Fri Feb 09 2018 Fedora Release Engineering - 3-26 - Rebuilt for https://fedoraproject.org/wiki/Fedora_28_Mass_Rebuild From 6cc8000e1491ed61cdaedbb9762ba225111c6000 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Fri, 6 Apr 2018 11:48:08 +0200 Subject: [PATCH 026/218] Fix the %py_dist_name macro ..to not convert dots (".") into dashes, so that submodules can be addressed as well Resolves: rhbz#1564095 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 38a71dc..397b4fa 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # Converts Python dist name to a canonical format %py_dist_name() %{lua:\ name = rpm.expand("%{?1:%{1}}");\ - canonical = string.gsub(string.lower(name), "%W+", "-");\ + canonical = string.gsub(string.lower(name), "[^%w%.]+", "-");\ print(canonical);\ } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8c095d9..b659771 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 27%{?dist} +Release: 28%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -69,6 +69,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Fri Apr 06 2018 Tomas Orsava - 3-28 +- Fix the %%py_dist_name macro to not convert dots (".") into dashes, so that + submodules can be addressed as well +Resolves: rhbz#1564095 + * Fri Mar 23 2018 Miro Hrončok - 3-27 - make LDFLAGS propagated whenever CFLAGS are From 739136717f1f441c7671d5e035fc01f4b264e4d4 Mon Sep 17 00:00:00 2001 From: dmalcolm Date: Thu, 28 Jan 2010 19:09:18 +0000 Subject: [PATCH 027/218] Introduce macros.pybytecompile --- macros.pybytecompile | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 macros.pybytecompile diff --git a/macros.pybytecompile b/macros.pybytecompile new file mode 100644 index 0000000..a8b79b2 --- /dev/null +++ b/macros.pybytecompile @@ -0,0 +1,12 @@ +# Note that the path could itself be a python file, or a directory + +# Python's compile_all module only works on directories, and requires a max +# recursion depth + +%py_byte_compile()\ +python_binary="%1"\ +bytecode_compilation_path="%2"\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ +\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ +%{nil} From bcfeb6a0b180eb63d5e6f35f96c6efa0f3f2c1a4 Mon Sep 17 00:00:00 2001 From: Matej Stuchlik Date: Wed, 25 Mar 2015 14:35:47 +0100 Subject: [PATCH 028/218] Update macros.pybytecompile to 3.4 --- macros.pybytecompile => macros.pybytecompile3.4 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename macros.pybytecompile => macros.pybytecompile3.4 (100%) diff --git a/macros.pybytecompile b/macros.pybytecompile3.4 similarity index 100% rename from macros.pybytecompile rename to macros.pybytecompile3.4 From 99ca83f0546d513cf61fa070f0397287a8ad6aa1 Mon Sep 17 00:00:00 2001 From: Robert Kuska Date: Wed, 23 Sep 2015 11:30:57 +0200 Subject: [PATCH 029/218] Rename macros to 3.5 --- macros.pybytecompile3.4 => macros.pybytecompile3.5 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename macros.pybytecompile3.4 => macros.pybytecompile3.5 (100%) diff --git a/macros.pybytecompile3.4 b/macros.pybytecompile3.5 similarity index 100% rename from macros.pybytecompile3.4 rename to macros.pybytecompile3.5 From 3af7df6454777099b3397feb49c250040b9aba05 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Mon, 12 Sep 2016 14:45:49 +0200 Subject: [PATCH 030/218] Update %py_byte_compile macro --- macros.pybytecompile3.5 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/macros.pybytecompile3.5 b/macros.pybytecompile3.5 index a8b79b2..96d1826 100644 --- a/macros.pybytecompile3.5 +++ b/macros.pybytecompile3.5 @@ -6,7 +6,5 @@ %py_byte_compile()\ python_binary="%1"\ bytecode_compilation_path="%2"\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ -\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ %{nil} From 5208f057c143b26f6806be3d7a355f42c5fc7e26 Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Tue, 27 Sep 2016 21:06:04 +0200 Subject: [PATCH 031/218] Update macros.pybytecompile to 3.6 Rename the macros.pybytecompile3.5 file to macros.pybytecompile3.6 --- macros.pybytecompile3.5 => macros.pybytecompile3.6 | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename macros.pybytecompile3.5 => macros.pybytecompile3.6 (100%) diff --git a/macros.pybytecompile3.5 b/macros.pybytecompile3.6 similarity index 100% rename from macros.pybytecompile3.5 rename to macros.pybytecompile3.6 From 2e3d37231004aed7b32827ebdf918870ca9d7ffc Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 21 Mar 2017 11:51:35 +0100 Subject: [PATCH 032/218] Fix syntax error in %py_byte_compile macro (rhbz#1433569) --- macros.pybytecompile3.6 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros.pybytecompile3.6 b/macros.pybytecompile3.6 index 96d1826..3968c6e 100644 --- a/macros.pybytecompile3.6 +++ b/macros.pybytecompile3.6 @@ -6,5 +6,5 @@ %py_byte_compile()\ python_binary="%1"\ bytecode_compilation_path="%2"\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ %{nil} From 438faf7c86d1888cdbaaee9044ef7abffa3983f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 18 Jan 2018 16:53:05 +0100 Subject: [PATCH 033/218] Fix the py_byte_compile macro to work on Python 2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://bugzilla.redhat.com/show_bug.cgi?id=1484993 Inspired by Terje Røsten's workaround from that bugzilla --- macros.pybytecompile3.6 | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/macros.pybytecompile3.6 b/macros.pybytecompile3.6 index 3968c6e..f319979 100644 --- a/macros.pybytecompile3.6 +++ b/macros.pybytecompile3.6 @@ -3,8 +3,23 @@ # Python's compile_all module only works on directories, and requires a max # recursion depth +# Note that the py_byte_compile macro should work for python2 as well +# Which unfortunately makes the definition more complicated than it should be +# The condition should be reversed once /usr/bin/python is python3! + %py_byte_compile()\ -python_binary="%1"\ -bytecode_compilation_path="%2"\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ +py2_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ +}\ +\ +py3_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ +}\ +\ +[[ "%1" == *python3* ]] || py2_byte_compile "%1" "%2" && py3_byte_compile "%1" "%2" \ %{nil} From 21ac2a96538b95023d63f4e8657801d555a468a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 23 Mar 2018 13:44:27 +0100 Subject: [PATCH 034/218] Make the pybytecompile version agnostic again It is no longer bound to a specific pythonX package --- macros.pybytecompile3.6 => macros.pybytecompile | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename macros.pybytecompile3.6 => macros.pybytecompile (100%) diff --git a/macros.pybytecompile3.6 b/macros.pybytecompile similarity index 100% rename from macros.pybytecompile3.6 rename to macros.pybytecompile From 3c79d6a8993e288416e4a40c8b97638be76a43ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 23 Mar 2018 13:52:23 +0100 Subject: [PATCH 035/218] Move macros.pybytecompile from python3-devel - https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation - No conflicts need to be added the file is renamed - If both files are present, there is no harm, so we'll just remove it from 3.7 --- python-rpm-macros.spec | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b659771..8115155 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 28%{?dist} +Release: 29%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -8,6 +8,7 @@ Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 +Source4: macros.pybytecompile BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -51,12 +52,13 @@ RPM macros for building Python 3 packages. %install mkdir -p %{buildroot}/%{rpmmacrodir} -install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ +install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %{buildroot}/%{rpmmacrodir}/ %files %{rpmmacrodir}/macros.python +%{rpmmacrodir}/macros.pybytecompile %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm @@ -69,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Wed Apr 18 2018 Miro Hrončok - 3-29 +- move macros.pybytecompile from python3-devel + * Fri Apr 06 2018 Tomas Orsava - 3-28 - Fix the %%py_dist_name macro to not convert dots (".") into dashes, so that submodules can be addressed as well From 37a004eed88a71503a497bdfda6e830ffec2e353 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Mon, 18 Jun 2018 12:54:12 -0500 Subject: [PATCH 036/218] Add %pypi_source macro. Adds the %pypi_source macro, as well as %__pypi_url and %__pypi_default_extension. This should make references to sources in PyPI much simpler for nearly all Python packages. --- macros.python-srpm | 44 ++++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 6 +++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 397b4fa..c4bdd1c 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -61,3 +61,47 @@ end\ } +# Macro to replace overly complicated references to PyPI source files. +# Expands to the pythonhosted URL for a package +# Accepts zero to three arguments: +# 1: The PyPI project name, defaulting to %srcname if it is defined, then +# %pypi_name if it is defined, then just %name. +# 2: The PYPI version, defaulting to %version. +# 3: The file extension, defaulting to "tar.gz". (A period will be added +# automatically.) +# Requires %__pypi_url and %__pypi_default_extension to be defined. +%__pypi_url https://files.pythonhosted.org/packages/source/ +%__pypi_default_extension tar.gz + +%pypi_source() %{lua: + local src = rpm.expand('%1') + local ver = rpm.expand('%2') + local ext = rpm.expand('%3') + local url = rpm.expand('%__pypi_url') +\ + -- If no first argument, try %srcname, then %pypi_name, then %name + -- Note that rpm leaves macros unchanged if they are not defined. + if src == '%1' then + src = rpm.expand('%srcname') + end + if src == '%srcname' then + src = rpm.expand('%pypi_name') + end + if src == '%pypi_name' then + src = rpm.expand('%name') + end +\ + -- If no second argument, use %version + if ver == '%2' then + ver = rpm.expand('%version') + end +\ + -- If no third argument, use the preset default extension + if ext == '%3' then + ext = rpm.expand('%__pypi_default_extension') + end +\ + local first = string.sub(src, 1, 1) +\ + print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext) +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8115155..fb4e8d3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 29%{?dist} +Release: 30%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Mon Jun 18 2018 Jason L Tibbitts III - 3-30 +- Add %%pypi_source macro, as well as %%__pypi_url and + %%_pypi_default_extension. + * Wed Apr 18 2018 Miro Hrončok - 3-29 - move macros.pybytecompile from python3-devel From 133ccf712356b3645831d61aa66a3d7476948841 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Mon, 18 Jun 2018 18:38:44 -0500 Subject: [PATCH 037/218] Add %pypi_source macro. --- macros.python-srpm | 45 ++++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 274e1c8..82a14ed 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -7,3 +7,48 @@ %__python3_other /bin/true %py3_other_build /bin/true %py3_other_install /bin/true + +# Macro to replace overly complicated references to PyPI source files. +# Expands to the pythonhosted URL for a package +# Accepts zero to three arguments: +# 1: The PyPI project name, defaulting to %srcname if it is defined, then +# %pypi_name if it is defined, then just %name. +# 2: The PYPI version, defaulting to %version. +# 3: The file extension, defaulting to "tar.gz". (A period will be added +# automatically.) +# Requires %__pypi_url and %__pypi_default_extension to be defined. +%__pypi_url https://files.pythonhosted.org/packages/source/ +%__pypi_default_extension tar.gz + +%pypi_source() %{lua: + local src = rpm.expand('%1') + local ver = rpm.expand('%2') + local ext = rpm.expand('%3') + local url = rpm.expand('%__pypi_url') +\ + -- If no first argument, try %srcname, then %pypi_name, then %name + -- Note that rpm leaves macros unchanged if they are not defined. + if src == '%1' then + src = rpm.expand('%srcname') + end + if src == '%srcname' then + src = rpm.expand('%pypi_name') + end + if src == '%pypi_name' then + src = rpm.expand('%name') + end +\ + -- If no second argument, use %version + if ver == '%2' then + ver = rpm.expand('%version') + end +\ + -- If no third argument, use the preset default extension + if ext == '%3' then + ext = rpm.expand('%__pypi_default_extension') + end +\ + local first = string.sub(src, 1, 1) +\ + print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext) +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index f2c2bff..ee9a965 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 11%{?dist} +Release: 12%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -63,6 +63,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Mon Jun 18 2018 Jason L Tibbitts III - 3-12 +- Add %%pypi_source macro. + * Thu Sep 15 2016 Jason L Tibbitts III - 3-11 - Use %%rpmmacrodir to put the macros in the proper location. - Change %%python2_version* to a form that works on EL6. From 1ea9947b6fd6e4d049416cd8c67b59e2cff8cf75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 3 Jul 2018 14:17:33 +0200 Subject: [PATCH 038/218] Add %python3_platform useful for PYTHONPATH on arched builds This is mostly used when building Sphinx docs on arched build. Previously: PLATFORM=$(python3 -c "import sysconfig; print(sysconfig.get_platform())") export PYTHONPATH=../build/lib.${PLATFORM}-%{python3_version} make man Or: PYTHONPATH=`realpath ../build/lib.linux*` make Or: PYTHONPATH=$(echo $PWD/build/lib.linux-*) make html Now: PYTHONPATH=../build/lib.%{python3_platform}-%{python3_version} --- macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index 79c33f1..25c6f62 100644 --- a/macros.python3 +++ b/macros.python3 @@ -3,6 +3,7 @@ %python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3])") %python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") +%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index fb4e8d3..413bbc1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 30%{?dist} +Release: 31%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Tue Jul 03 2018 Miro Hrončok - 3-31 +- Add %%python3_platform useful for PYTHONPATH on arched builds + * Mon Jun 18 2018 Jason L Tibbitts III - 3-30 - Add %%pypi_source macro, as well as %%__pypi_url and %%_pypi_default_extension. From 5e1687a64cdd279cf27a6d981fba5936a84d9994 Mon Sep 17 00:00:00 2001 From: Jason Tibbitts Date: Mon, 9 Jul 2018 14:34:36 -0500 Subject: [PATCH 039/218] Backport %python3_platform macro. --- macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index 2ef1fe7..7226ccc 100644 --- a/macros.python3 +++ b/macros.python3 @@ -5,6 +5,7 @@ %python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3])") %python3_version_nodots %(%{__python3} -c "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") +%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ee9a965..bef8854 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 12%{?dist} +Release: 13%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -63,6 +63,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Mon Jul 09 2018 Jason L Tibbitts III - 3-13 +- Backport %%python3_platform macro. + * Mon Jun 18 2018 Jason L Tibbitts III - 3-12 - Add %%pypi_source macro. From f961fbbc2948294bcf95b5a57aafe923d6e49eba Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 10 Jul 2018 15:02:20 +0200 Subject: [PATCH 040/218] Fix %%py_byte_compile macro When invoked with a Python 2 binary it also mistakenly ran py3_byte_compile. --- macros.pybytecompile | 2 +- python-rpm-macros.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index f319979..390d4f7 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -21,5 +21,5 @@ py3_byte_compile () {\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ }\ \ -[[ "%1" == *python3* ]] || py2_byte_compile "%1" "%2" && py3_byte_compile "%1" "%2" \ +[[ "%1" != *python3* ]] && py2_byte_compile "%1" "%2" || py3_byte_compile "%1" "%2" \ %{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 413bbc1..8a239af 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 31%{?dist} +Release: 32%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Tue Jul 10 2018 Tomas Orsava - 3-32 +- Fix %%py_byte_compile macro: when invoked with a Python 2 binary it also + mistakenly ran py3_byte_compile + * Tue Jul 03 2018 Miro Hrončok - 3-31 - Add %%python3_platform useful for PYTHONPATH on arched builds From 967bb3c12ed8bb1f4cbe3d02ab80aa8c79350521 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 14 Jul 2018 01:58:10 +0000 Subject: [PATCH 041/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8a239af..545e2c3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 32%{?dist} +Release: 33%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Sat Jul 14 2018 Fedora Release Engineering - 3-33 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild + * Tue Jul 10 2018 Tomas Orsava - 3-32 - Fix %%py_byte_compile macro: when invoked with a Python 2 binary it also mistakenly ran py3_byte_compile From 05333eb5866a865282e37ad4d8d24f0b2b6cabb1 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 11 Jul 2018 15:11:52 +0200 Subject: [PATCH 042/218] macros.pybytecompile: Detect Python version through sys.version_info ...instead of guessing from the executable name. This should make it work on EPEL7 as well where we ship 3.4 and 3.6 https://bugzilla.redhat.com/show_bug.cgi?id=1599809 --- macros.pybytecompile | 8 +++++--- python-rpm-macros.spec | 6 +++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index 390d4f7..f5439cf 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -3,9 +3,8 @@ # Python's compile_all module only works on directories, and requires a max # recursion depth -# Note that the py_byte_compile macro should work for python2 as well +# Note that the py_byte_compile macro should work for all Python versions # Which unfortunately makes the definition more complicated than it should be -# The condition should be reversed once /usr/bin/python is python3! %py_byte_compile()\ py2_byte_compile () {\ @@ -21,5 +20,8 @@ py3_byte_compile () {\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ }\ \ -[[ "%1" != *python3* ]] && py2_byte_compile "%1" "%2" || py3_byte_compile "%1" "%2" \ +# Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ +python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ +# The bytecompilation syntax has changed between Python 3.4 and Python 3.5, so for 3.4 and earlier we use the "Python 2" syntax \ +[ "$python_version" -ge 35 ] && py3_byte_compile "%1" "%2" || py2_byte_compile "%1" "%2" \ %{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 545e2c3..86949e6 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 33%{?dist} +Release: 34%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Sat Jul 14 2018 Tomas Orsava - 3-34 +- macros.pybytecompile: Detect Python version through sys.version_info instead + of guessing from the executable name + * Sat Jul 14 2018 Fedora Release Engineering - 3-33 - Rebuilt for https://fedoraproject.org/wiki/Fedora_29_Mass_Rebuild From d3d040818d53e2c1bdb03a9e089773de92dca0e7 Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Sat, 28 Jul 2018 14:21:13 +0200 Subject: [PATCH 043/218] Change way how enabling-depgen works internally Signed-off-by: Igor Gnatenko --- macros.python | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python b/macros.python index f0039c8..4107d68 100644 --- a/macros.python +++ b/macros.python @@ -77,5 +77,5 @@ } %python_enable_dependency_generator() \ -%global __python_requires %{_rpmconfigdir}/pythondistdeps.py --requires \ +%global __pythondist_requires %{_rpmconfigdir}/pythondistdeps.py --requires \ %{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 86949e6..934a232 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 34%{?dist} +Release: 35%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Sat Jul 28 2018 Igor Gnatenko - 3-35 +- Change way how enabling-depgen works internally + * Sat Jul 14 2018 Tomas Orsava - 3-34 - macros.pybytecompile: Detect Python version through sys.version_info instead of guessing from the executable name From 90b0bf74804e4b20ce2bad9bd1ed61ff1f0d1e3d Mon Sep 17 00:00:00 2001 From: Pavel Raiskup Date: Fri, 3 Aug 2018 17:08:55 +0200 Subject: [PATCH 044/218] macros.python: drop last %__python2 occurrence This is needed so people can do things like: %if %{with python3} %global __python %__python3 %else %global __python %__python2 %else And than just use %__python, %py_build, %py_install, %python_sitelib, ... --- macros.python | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 4107d68..70eb148 100644 --- a/macros.python +++ b/macros.python @@ -5,7 +5,7 @@ # the macro %py_build() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?*} + %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} sleep 1 } From 25b297c00670522356fcda2532090f15dc7cfb46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 15 Aug 2018 11:56:37 +0200 Subject: [PATCH 045/218] Bump for 90b0bf7 --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 934a232..bd849d7 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 35%{?dist} +Release: 36%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Wed Aug 15 2018 Miro Hrončok - 3-36 +- Make %%py_build wokr if %%__python is defined to custom value + * Sat Jul 28 2018 Igor Gnatenko - 3-35 - Change way how enabling-depgen works internally From 8f067ff3858ac0456fbfe0f99c5df22a3ecab628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 15 Aug 2018 18:26:37 +0200 Subject: [PATCH 046/218] Make %py_byte_compile terminate build on SyntaxErrors (#1616219) Also, make it simpler again --- macros.pybytecompile | 32 ++++++++++++-------------------- python-rpm-macros.spec | 5 ++++- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index f5439cf..a41b3a2 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -3,25 +3,17 @@ # Python's compile_all module only works on directories, and requires a max # recursion depth -# Note that the py_byte_compile macro should work for all Python versions -# Which unfortunately makes the definition more complicated than it should be +# Usage: +# %py_byte_compile +# Example: +# %py_byte_compile %{__python3} %{buildroot}%{_datadir}/spam/plugins/ + +# This will terminate build on SyntaxErrors, if you want to avoid that, +# use it in a subshell like this: +# (%{py_byte_compile }) || : %py_byte_compile()\ -py2_byte_compile () {\ - python_binary="%1"\ - bytecode_compilation_path="%2"\ - find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ - find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ -}\ -\ -py3_byte_compile () {\ - python_binary="%1"\ - bytecode_compilation_path="%2"\ - find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ -}\ -\ -# Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ -python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ -# The bytecompilation syntax has changed between Python 3.4 and Python 3.5, so for 3.4 and earlier we use the "Python 2" syntax \ -[ "$python_version" -ge 35 ] && py3_byte_compile "%1" "%2" || py2_byte_compile "%1" "%2" \ -%{nil} +python_binary="%1"\ +bytecode_compilation_path="%2"\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -m py_compile\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -m py_compile diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bd849d7..e3d2d85 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 36%{?dist} +Release: 37%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -71,6 +71,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Wed Aug 15 2018 Miro Hrončok - 3-37 +- Make %%py_byte_compile terminate build on SyntaxErrors (#1616219) + * Wed Aug 15 2018 Miro Hrončok - 3-36 - Make %%py_build wokr if %%__python is defined to custom value From beaa2eec4f426655773cc89eb1331f117757c8a9 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Thu, 20 Sep 2018 11:59:50 +0200 Subject: [PATCH 047/218] Move the __python2/3 macros to the python-srpm-macros subpackage This facilitates using the %%{__python2/3} in Build/Requires --- macros.python-srpm | 7 +++++++ macros.python2 | 1 - macros.python3 | 1 - python-rpm-macros.spec | 8 +++++++- 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index c4bdd1c..514a449 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -1,3 +1,10 @@ +# Define the Python interpreter paths in the SRPM macros so that +# - they can be used in Build/Requires +# - they can be used in non-Python packages where requiring pythonX-devel would +# be an overkill +%__python2 /usr/bin/python2 +%__python3 /usr/bin/python3 + # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 diff --git a/macros.python2 b/macros.python2 index d6c74c2..4b1390c 100644 --- a/macros.python2 +++ b/macros.python2 @@ -1,4 +1,3 @@ -%__python2 /usr/bin/python2 %python2_sitelib %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python2_sitearch %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python2_version %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") diff --git a/macros.python3 b/macros.python3 index 25c6f62..44e2889 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,4 +1,3 @@ -%__python3 /usr/bin/python3 %python3_sitelib %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3])") diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e3d2d85..649cd30 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 37%{?dist} +Release: 38%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -31,6 +31,7 @@ RPM macros for building Python source packages. %package -n python2-rpm-macros Summary: RPM macros for building Python 2 packages +Requires: python-srpm-macros >= 3-38 # Would need to be different for each release - worth it? #Conflicts: python2-devel < 2.7.11-3 @@ -39,6 +40,7 @@ RPM macros for building Python 2 packages. %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages +Requires: python-srpm-macros >= 3-38 # Would need to be different for each release - worth it? #Conflicts: python3-devel < 3.5.1-3 @@ -71,6 +73,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Thu Sep 20 2018 Tomas Orsava - 3-38 +- Move the __python2/3 macros to the python-srpm-macros subpackage +- This facilitates using the %%{__python2/3} in Build/Requires + * Wed Aug 15 2018 Miro Hrončok - 3-37 - Make %%py_byte_compile terminate build on SyntaxErrors (#1616219) From c8932dcbef98dd5e2630b92be7c773b72166dd70 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 Nov 2018 11:06:22 +0100 Subject: [PATCH 048/218] Move "sleep 1" workaround from py3_build to py2_build https://bugzilla.redhat.com/show_bug.cgi?id=1644923 --- macros.python2 | 5 +++++ macros.python3 | 3 --- python-rpm-macros.spec | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/macros.python2 b/macros.python2 index 4b1390c..38c6eb9 100644 --- a/macros.python2 +++ b/macros.python2 @@ -7,19 +7,24 @@ # Use the slashes after expand so that the command starts on the same line as # the macro +# The `sleep 1` commands work around a race in install; see: +# https://bugzilla.redhat.com/show_bug.cgi?id=1644923 %py2_build() %{expand:\\\ + sleep 1 CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} sleep 1 } %py2_build_egg() %{expand:\\\ + sleep 1 CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} sleep 1 } %py2_build_wheel() %{expand:\\\ + sleep 1 CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} sleep 1 diff --git a/macros.python3 b/macros.python3 index 44e2889..1952aed 100644 --- a/macros.python3 +++ b/macros.python3 @@ -12,19 +12,16 @@ %py3_build() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} - sleep 1 } %py3_build_egg() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} - sleep 1 } %py3_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} - sleep 1 } %py3_install() %{expand:\\\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 649cd30..1d26b45 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 38%{?dist} +Release: 39%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Thu Nov 01 2018 Petr Viktorin - 3-39 +- Move "sleep 1" workaround from py3_build to py2_build (#1644923) + * Thu Sep 20 2018 Tomas Orsava - 3-38 - Move the __python2/3 macros to the python-srpm-macros subpackage - This facilitates using the %%{__python2/3} in Build/Requires From 4b3c23b2337df1512b0eff3fe10fb96a8dc9454f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 5 Dec 2018 15:35:41 +0100 Subject: [PATCH 049/218] Workaround leaking buildroot PATH in %py_byte_compile (#1647212) --- macros.pybytecompile | 11 ++++++++--- python-rpm-macros.spec | 5 ++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index a41b3a2..d06071e 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -14,6 +14,11 @@ %py_byte_compile()\ python_binary="%1"\ -bytecode_compilation_path="%2"\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -m py_compile\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -m py_compile +buildroot_path="%2"\ +bytecode_compilation_path=".${buildroot_path/#$RPM_BUILD_ROOT}"\ +failure=0\ +pushd $RPM_BUILD_ROOT\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -m py_compile || failure=1\ +find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -m py_compile || failure=1\ +popd\ +test $failure -eq 0 diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 1d26b45..44639b8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 39%{?dist} +Release: 40%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Wed Dec 05 2018 Miro Hrončok - 3-40 +- Workaround leaking buildroot PATH in %py_byte_compile (#1647212) + * Thu Nov 01 2018 Petr Viktorin - 3-39 - Move "sleep 1" workaround from py3_build to py2_build (#1644923) From 9b8fac037df29067489914477c388574845cb5e5 Mon Sep 17 00:00:00 2001 From: Igor Gnatenko Date: Thu, 20 Dec 2018 14:35:02 +0100 Subject: [PATCH 050/218] Add %python_disable_dependency_generator Signed-off-by: Igor Gnatenko --- macros.python | 4 ++-- python-rpm-macros.spec | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/macros.python b/macros.python index 70eb148..8ad1161 100644 --- a/macros.python +++ b/macros.python @@ -76,6 +76,6 @@ end } -%python_enable_dependency_generator() \ -%global __pythondist_requires %{_rpmconfigdir}/pythondistdeps.py --requires \ +%python_disable_dependency_generator() \ +%undefine __pythondist_requires \ %{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 44639b8..727b632 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 40%{?dist} +Release: 41%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Thu Dec 20 2018 Igor Gnatenko - 3-41 +- Add %%python_disable_dependency_generator + * Wed Dec 05 2018 Miro Hrončok - 3-40 - Workaround leaking buildroot PATH in %py_byte_compile (#1647212) From 813a86fcc6b73e1ec7955e27e7789fb78761518a Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 2 Feb 2019 09:09:45 +0000 Subject: [PATCH 051/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 727b632..b2b04bd 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 41%{?dist} +Release: 42%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Sat Feb 02 2019 Fedora Release Engineering - 3-42 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild + * Thu Dec 20 2018 Igor Gnatenko - 3-41 - Add %%python_disable_dependency_generator From 0fa5a2d81130bd5428f91d766a172192c808a3c8 Mon Sep 17 00:00:00 2001 From: Carl George Date: Wed, 6 Mar 2019 08:11:49 -0600 Subject: [PATCH 052/218] Move macros.pybytecompile in here from python3X-devel Backporting to el6 branch from #10 (epel7). --- macros.pybytecompile | 27 +++++++++++++++++++++++++++ python-rpm-macros.spec | 11 +++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 macros.pybytecompile diff --git a/macros.pybytecompile b/macros.pybytecompile new file mode 100644 index 0000000..f5439cf --- /dev/null +++ b/macros.pybytecompile @@ -0,0 +1,27 @@ +# Note that the path could itself be a python file, or a directory + +# Python's compile_all module only works on directories, and requires a max +# recursion depth + +# Note that the py_byte_compile macro should work for all Python versions +# Which unfortunately makes the definition more complicated than it should be + +%py_byte_compile()\ +py2_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2]) for f in sys.argv[1:]]' || :\ +}\ +\ +py3_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("$RPM_BUILD_ROOT")[2], optimize=opt) for opt in range(2) for f in sys.argv[1:]]' || :\ +}\ +\ +# Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ +python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ +# The bytecompilation syntax has changed between Python 3.4 and Python 3.5, so for 3.4 and earlier we use the "Python 2" syntax \ +[ "$python_version" -ge 35 ] && py3_byte_compile "%1" "%2" || py2_byte_compile "%1" "%2" \ +%{nil} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bef8854..c4ad781 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 13%{?dist} +Release: 14%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -8,6 +8,7 @@ Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 +Source5: macros.pybytecompile BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -45,12 +46,13 @@ RPM macros for building Python 3 packages. %install mkdir -p %{buildroot}/%{rpmmacrodir} -install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ +install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE5} \ %{buildroot}/%{rpmmacrodir} %files %{rpmmacrodir}/macros.python +%{rpmmacrodir}/macros.pybytecompile %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm @@ -63,6 +65,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} \ %changelog +* Wed Mar 06 2019 Carl George - 3-14 +- Move macros.pybytecompile in here from python3X-devel +- macros.pybytecompile: Detect Python version through sys.version_info instead + of guessing from the executable name + * Mon Jul 09 2018 Jason L Tibbitts III - 3-13 - Backport %%python3_platform macro. From d38048d54daf71dbd4075e8d14725913ad049705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 10 Jun 2019 14:45:52 +0200 Subject: [PATCH 053/218] Define %python_sitelib, %python_sitearch, %python_version, %python_version_nodots In rpm 4.15 those are no longer defined. The meaning of "python" is derived from %__python - and that errors by default unless user defined. Example usage: %global __python /usr/bin/pypy3 ... %files %{python_sitelib}/foo/ --- macros.python | 7 +++++++ python-rpm-macros.spec | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 8ad1161..6f81850 100644 --- a/macros.python +++ b/macros.python @@ -1,3 +1,10 @@ +# unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 +%__python %{error:attempt to use unversioned python, define %%__python to %{_bindir}/python2 or %{_bindir}/python3 explicitly} +%python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") +%python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + %py_setup setup.py %py_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b2b04bd..3b213c3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 42%{?dist} +Release: 43%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,11 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Mon Jun 10 2019 Miro Hrončok - 3-43 +- Define %%python_sitelib, %%python_sitearch, %%python_version, %%python_version_nodots, + in rpm 4.15 those are no longer defined, the meaning of python is derived from %%__python. +- Usage of %%__python or the above-mentioned macros will error unless user defined. + * Sat Feb 02 2019 Fedora Release Engineering - 3-42 - Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild From 536b2efe4eb604493cc53b7c4617a34ac261d290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 12 Jun 2019 11:26:03 +0200 Subject: [PATCH 054/218] Move %__python definition to the srpm macros, so it is always present --- macros.python | 2 +- macros.python-srpm | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 6f81850..0e9c419 100644 --- a/macros.python +++ b/macros.python @@ -1,5 +1,5 @@ # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 -%__python %{error:attempt to use unversioned python, define %%__python to %{_bindir}/python2 or %{_bindir}/python3 explicitly} +# __python is defined to error by default in the srpm macros %python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") diff --git a/macros.python-srpm b/macros.python-srpm index 514a449..f25189b 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -5,6 +5,10 @@ %__python2 /usr/bin/python2 %__python3 /usr/bin/python3 +# This now errors unless redefined to user provided value +%__python %{error:attempt to use unversioned python, define %%__python to /usr/bin/python2 or /usr/bin/python3 explicitly} + + # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 From 04769fa014fe51d623c5b2e110d36bd73d035839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 17 Jun 2019 09:34:06 +0200 Subject: [PATCH 055/218] Remove the arched provide from %python_provide macro The way it fetched the information abut archfulness was not reliable and will stop working entirely. See https://bugzilla.redhat.com/show_bug.cgi?id=1705656 --- macros.python | 4 ---- python-rpm-macros.spec | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/macros.python b/macros.python index 0e9c419..266f356 100644 --- a/macros.python +++ b/macros.python @@ -49,10 +49,6 @@ package = rpm.expand("%{?1}") vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") if (string.starts(package, "python2-")) then - if (rpm.expand("%{?buildarch}") ~= "noarch") then - str = "Provides: python-" .. string.sub(package,9,string.len(package)) .. "%{?_isa} = " .. vr - print(rpm.expand(str)) - end print("\\nProvides: python-") print(string.sub(package,9,string.len(package))) print(" = ") diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3b213c3..c47f6de 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -77,6 +77,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ - Define %%python_sitelib, %%python_sitearch, %%python_version, %%python_version_nodots, in rpm 4.15 those are no longer defined, the meaning of python is derived from %%__python. - Usage of %%__python or the above-mentioned macros will error unless user defined. +- The %%python_provide macro no longer gives the arched provide for arched packages (#1705656) * Sat Feb 02 2019 Fedora Release Engineering - 3-42 - Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild From cf8051e7f5ec584c857c86e23e5f4a38c2a27350 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 17 Jun 2019 15:33:25 +0200 Subject: [PATCH 056/218] Make %__python /usr/bin/python once again until we are ready See https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/22#comment-26552 and further. --- macros.python-srpm | 5 +++-- python-rpm-macros.spec | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f25189b..fb960ea 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -5,8 +5,9 @@ %__python2 /usr/bin/python2 %__python3 /usr/bin/python3 -# This now errors unless redefined to user provided value -%__python %{error:attempt to use unversioned python, define %%__python to /usr/bin/python2 or /usr/bin/python3 explicitly} +# For backwards compatibility only +# See the comments in https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/22 +%__python /usr/bin/python # python3_pkgversion specifies the version of Python 3 in the distro. It can be diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c47f6de..170092b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 43%{?dist} +Release: 44%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Mon Jun 17 2019 Miro Hrončok - 3-44 +- Make %%__python /usr/bin/python once again until we are ready + * Mon Jun 10 2019 Miro Hrončok - 3-43 - Define %%python_sitelib, %%python_sitearch, %%python_version, %%python_version_nodots, in rpm 4.15 those are no longer defined, the meaning of python is derived from %%__python. From b67b47d5b0a85fcf10371184446d09eaf014cfca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 9 Jul 2019 13:40:59 +0200 Subject: [PATCH 057/218] %python_provide: Don't try to obsolete %_isa provides Based on recent changes in 04769fa014fe51d623c5b2e110d36bd73d035839, packagers might want to use: %{?python_provide:%python_provide python2-foo%{?_isa}} ...for backwards compatibility. However the macro adds obsoletes and since RPM 4.15, obsoletes with %{?_isa} are not possible: Only package names are allowed in Obsoletes: Obsoletes: python-foo(x86-64) < ... To allow such usage, %python_provide now only obsoletes if the argument does not end with ")". --- macros.python | 12 +++++++----- python-rpm-macros.spec | 5 ++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/macros.python b/macros.python index 266f356..0ac2b82 100644 --- a/macros.python +++ b/macros.python @@ -53,11 +53,13 @@ print(string.sub(package,9,string.len(package))) print(" = ") print(vr) - --Obsoleting the previous default python package - print("\\nObsoletes: python-") - print(string.sub(package,9,string.len(package))) - print(" < ") - print(vr) + --Obsoleting the previous default python package (if it doesn't have isa) + if (string.sub(package, "-1") ~= ")") then + print("\\nObsoletes: python-") + print(string.sub(package,9,string.len(package))) + print(" < ") + print(vr) + end elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then --No unversioned provides as python3 is not default elseif (rpm.expand("%{?python3_other_pkgversion}") ~= "" and string.starts(package, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 170092b..5c11c3b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 44%{?dist} +Release: 45%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Tue Jul 09 2019 Miro Hrončok - 3-45 +- %%python_provide: Don't try to obsolete %%_isa provides + * Mon Jun 17 2019 Miro Hrončok - 3-44 - Make %%__python /usr/bin/python once again until we are ready From 64119cef2c7087829de4d2af897067375af55339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 12 Jul 2019 16:36:27 +0200 Subject: [PATCH 058/218] Switch %python_provide behavior between Python 2 <--> 3 https://fedoraproject.org/wiki/Changes/Python_means_Python3 Welcome to the future. --- macros.python | 17 ++++++++--------- python-rpm-macros.spec | 6 +++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/macros.python b/macros.python index 0ac2b82..e431b92 100644 --- a/macros.python +++ b/macros.python @@ -49,31 +49,30 @@ package = rpm.expand("%{?1}") vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") if (string.starts(package, "python2-")) then + --No unversioned provides as python2 is not default + elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then print("\\nProvides: python-") - print(string.sub(package,9,string.len(package))) + l = 8 + string.len(rpm.expand("%{python3_pkgversion}")) + print(string.sub(package,l,string.len(package))) print(" = ") print(vr) --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") - print(string.sub(package,9,string.len(package))) + print(string.sub(package,l,string.len(package))) print(" < ") print(vr) end - elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then - --No unversioned provides as python3 is not default elseif (rpm.expand("%{?python3_other_pkgversion}") ~= "" and string.starts(package, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then --No unversioned provides as python3_other is not default elseif (string.starts(package, "pypy-")) then --No unversioned provides as pypy is not default + elseif (string.starts(package, "pypy2-")) then + --No unversioned provides as pypy is not default elseif (string.starts(package, "pypy3-")) then --No unversioned provides as pypy is not default elseif (string.starts(package, "python-")) then - --Providing the current default python - print("Provides: python2-") - print(string.sub(package,8,string.len(package))) - print(" = ") - print(vr) + --No unversioned provides needed for unversioned python else print("%python_provide: ERROR: ") print(package) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5c11c3b..75de388 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 45%{?dist} +Release: 46%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,10 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Fri Jul 12 2019 Miro Hrončok - 3-46 +- %%python_provide: Switch python2 and python3 behavior +- https://fedoraproject.org/wiki/Changes/Python_means_Python3 + * Tue Jul 09 2019 Miro Hrončok - 3-45 - %%python_provide: Don't try to obsolete %%_isa provides From 76681ad58e914924e4ffbd5ddc48359f871f68ba Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 11 Jul 2019 14:24:39 +0200 Subject: [PATCH 059/218] Use a new module compileall2 for Python byte-compilation --- compileall2.py | 464 +++++++++++++++++++++++++++++++++++++++++ macros.pybytecompile | 32 ++- python-rpm-macros.spec | 22 +- 3 files changed, 500 insertions(+), 18 deletions(-) create mode 100644 compileall2.py diff --git a/compileall2.py b/compileall2.py new file mode 100644 index 0000000..b2c1e97 --- /dev/null +++ b/compileall2.py @@ -0,0 +1,464 @@ +"""Module/script to byte-compile all .py files to .pyc files. + +When called as a script with arguments, this compiles the directories +given as arguments recursively; the -l option prevents it from +recursing into directories. + +Without arguments, if compiles all modules on sys.path, without +recursing into subdirectories. (Even though it should do so for +packages -- for now, you'll have to deal with packages separately.) + +See module py_compile for details of the actual byte-compilation. +""" +import os +import sys +import importlib.util +import py_compile +import struct + +from functools import partial +from pathlib import Path + +# Python 3.7 and higher +PY37 = sys.version_info[0:2] >= (3, 7) +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) +# Python 3.5 and higher +PY35 = sys.version_info[0:2] >= (3, 5) + +# Python 3.7 and above has a different structure and length +# of pyc files header. Also, multiple ways how to invalidate pyc file was +# introduced in Python 3.7. These cases are covered by variables here or by PY37 +# variable itself. +if PY37: + pyc_struct_format = '<4sll' + pyc_header_lenght = 12 + pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) +else: + pyc_struct_format = '<4sl' + pyc_header_lenght = 8 + pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) + +RECURSION_LIMIT = sys.getrecursionlimit() + +__all__ = ["compile_dir","compile_file","compile_path"] + +def optimization_kwarg(opt): + """Returns opt as a dictionary {optimization: opt} for use as **kwarg + for Python >= 3.5 and empty dictionary for Python 3.4""" + if PY35: + return dict(optimization=opt) + else: + # `debug_override` is a way how to enable optimized byte-compiled files + # (.pyo) in Python <= 3.4 + if opt: + return dict(debug_override=False) + else: + return dict() + +def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) + +def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, + rx=None, quiet=0, legacy=False, optimize=-1, workers=1, + invalidation_mode=None, stripdir=None, + prependdir=None, limit_sl_dest=None): + """Byte-compile all modules in the given directory tree. + + Arguments (only dir is required): + + dir: the directory to byte-compile + maxlevels: maximum recursion level (default `sys.getrecursionlimit()`) + ddir: the directory that will be prepended to the path to the + file as it is compiled into each byte-code file. + force: if True, force compilation, even if timestamps are up-to-date + quiet: full output with False or 0, errors only with 1, + no output with 2 + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. + workers: maximum number of parallel workers + invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path + """ + ProcessPoolExecutor = None + if workers is not None: + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + elif workers != 1: + try: + # Only import when needed, as low resource platforms may + # fail to import it + from concurrent.futures import ProcessPoolExecutor + except ImportError: + workers = 1 + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + workers = workers or None + with ProcessPoolExecutor(max_workers=workers) as executor: + results = executor.map(partial(compile_file, + ddir=ddir, force=force, + rx=rx, quiet=quiet, + legacy=legacy, + optimize=optimize, + invalidation_mode=invalidation_mode, + stripdir=stripdir, + prependdir=prependdir, + limit_sl_dest=limit_sl_dest), + files) + success = min(results, default=True) + else: + for file in files: + if not compile_file(file, ddir, force, rx, quiet, + legacy, optimize, invalidation_mode, + stripdir=stripdir, prependdir=prependdir, + limit_sl_dest=limit_sl_dest): + success = False + return success + +def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, + legacy=False, optimize=-1, + invalidation_mode=None, stripdir=None, prependdir=None, + limit_sl_dest=None): + """Byte-compile one file. + + Arguments (only fullname is required): + + fullname: the file to byte-compile + ddir: if given, the directory name compiled in to the + byte-code file. + force: if True, force compilation, even if timestamps are up-to-date + quiet: full output with False or 0, errors only with 1, + no output with 2 + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. + invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path. + """ + success = True + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + dfile = None + + if ddir is not None: + if not PY36: + ddir = str(ddir) + dfile = os.path.join(ddir, name) + + if stripdir is not None: + fullname_parts = fullname.split(os.path.sep) + stripdir_parts = stripdir.split(os.path.sep) + ddir_parts = list(fullname_parts) + + for spart, opart in zip(stripdir_parts, fullname_parts): + if spart == opart: + ddir_parts.remove(spart) + + dfile = os.path.join(*ddir_parts) + + if prependdir is not None: + if dfile is None: + dfile = os.path.join(prependdir, fullname) + else: + dfile = os.path.join(prependdir, dfile) + + if isinstance(optimize, int): + optimize = [optimize] + + if rx is not None: + mo = rx.search(fullname) + if mo: + return success + + if limit_sl_dest is not None and os.path.islink(fullname): + if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents: + return success + + opt_cfiles = {} + + if os.path.isfile(fullname): + for opt_level in optimize: + if legacy: + opt_cfiles[opt_level] = fullname + 'c' + else: + if opt_level >= 0: + opt = opt_level if opt_level >= 1 else '' + opt_kwarg = optimization_kwarg(opt) + cfile = (importlib.util.cache_from_source( + fullname, **opt_kwarg)) + opt_cfiles[opt_level] = cfile + else: + cfile = importlib.util.cache_from_source(fullname) + opt_cfiles[opt_level] = cfile + + head, tail = name[:-3], name[-3:] + if tail == '.py': + if not force: + try: + mtime = int(os.stat(fullname).st_mtime) + expect = struct.pack(*(pyc_header_format + (mtime,))) + for cfile in opt_cfiles.values(): + with open(cfile, 'rb') as chandle: + actual = chandle.read(pyc_header_lenght) + if expect != actual: + break + else: + return success + except OSError: + pass + if not quiet: + print('Compiling {!r}...'.format(fullname)) + try: + for opt_level, cfile in opt_cfiles.items(): + if PY37: + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=opt_level, + invalidation_mode=invalidation_mode) + else: + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=opt_level) + except py_compile.PyCompileError as err: + success = False + if quiet >= 2: + return success + elif quiet: + print('*** Error compiling {!r}...'.format(fullname)) + else: + print('*** ', end='') + # escape non-printable characters in msg + msg = err.msg.encode(sys.stdout.encoding, + errors='backslashreplace') + msg = msg.decode(sys.stdout.encoding) + print(msg) + except (SyntaxError, UnicodeError, OSError) as e: + success = False + if quiet >= 2: + return success + elif quiet: + print('*** Error compiling {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + else: + if ok == 0: + success = False + return success + +def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, + legacy=False, optimize=-1, + invalidation_mode=None): + """Byte-compile all module on sys.path. + + Arguments (all optional): + + skip_curdir: if true, skip current directory (default True) + maxlevels: max recursion level (default 0) + force: as for compile_dir() (default False) + quiet: as for compile_dir() (default 0) + legacy: as for compile_dir() (default False) + optimize: as for compile_dir() (default -1) + invalidation_mode: as for compiler_dir() + """ + success = True + for dir in sys.path: + if (not dir or dir == os.curdir) and skip_curdir: + if quiet < 2: + print('Skipping current directory') + else: + success = success and compile_dir( + dir, + maxlevels, + None, + force, + quiet=quiet, + legacy=legacy, + optimize=optimize, + invalidation_mode=invalidation_mode, + ) + return success + + +def main(): + """Script main program.""" + import argparse + + parser = argparse.ArgumentParser( + description='Utilities to support installing Python libraries.') + parser.add_argument('-l', action='store_const', const=0, + default=RECURSION_LIMIT, dest='maxlevels', + help="don't recurse into subdirectories") + parser.add_argument('-r', type=int, dest='recursion', + help=('control the maximum recursion level. ' + 'if `-l` and `-r` options are specified, ' + 'then `-r` takes precedence.')) + parser.add_argument('-f', action='store_true', dest='force', + help='force rebuild even if timestamps are up to date') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('-b', action='store_true', dest='legacy', + help='use legacy (pre-PEP3147) compiled file locations') + parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, + help=('directory to prepend to file paths for use in ' + 'compile-time tracebacks and in runtime ' + 'tracebacks in cases where the source file is ' + 'unavailable')) + parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir', + default=None, + help=('part of path to left-strip from path ' + 'to source file - for example buildroot. ' + 'if `-d` and `-s` options are specified, ' + 'then `-d` takes precedence.')) + parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir', + default=None, + help=('path to add as prefix to path ' + 'to source file - for example / to make ' + 'it absolute when some part is removed ' + 'by `-s` option' + 'if `-d` and `-a` options are specified, ' + 'then `-d` takes precedence.')) + parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, + help=('skip files matching the regular expression; ' + 'the regexp is searched for in the full path ' + 'of each file considered for compilation')) + parser.add_argument('-i', metavar='FILE', dest='flist', + help=('add all the files and directories listed in ' + 'FILE to the list considered for compilation; ' + 'if "-", names are read from stdin')) + parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', + help=('zero or more file and directory names ' + 'to compile; if no arguments given, defaults ' + 'to the equivalent of -l sys.path')) + parser.add_argument('-j', '--workers', default=1, + type=int, help='Run compileall concurrently') + parser.add_argument('-o', action='append', type=int, dest='opt_levels', + help=('Optimization levels to run compilation with.' + 'Default is -1 which uses optimization level of' + 'Python interpreter itself (specified by -O).')) + parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', + help='Ignore symlinks pointing outsite of the DIR') + + if PY37: + invalidation_modes = [mode.name.lower().replace('_', '-') + for mode in py_compile.PycInvalidationMode] + parser.add_argument('--invalidation-mode', + choices=sorted(invalidation_modes), + help=('set .pyc invalidation mode; defaults to ' + '"checked-hash" if the SOURCE_DATE_EPOCH ' + 'environment variable is set, and ' + '"timestamp" otherwise.')) + + args = parser.parse_args() + compile_dests = args.compile_dest + + if args.rx: + import re + args.rx = re.compile(args.rx) + + if args.limit_sl_dest == "": + args.limit_sl_dest = None + + if args.recursion is not None: + maxlevels = args.recursion + else: + maxlevels = args.maxlevels + + if args.opt_levels is None: + args.opt_levels = [-1] + + # if flist is provided then load it + if args.flist: + try: + with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + for line in f: + compile_dests.append(line.strip()) + except OSError: + if args.quiet < 2: + print("Error reading file list {}".format(args.flist)) + return False + + if args.workers is not None: + args.workers = args.workers or None + + if PY37 and args.invalidation_mode: + ivl_mode = args.invalidation_mode.replace('-', '_').upper() + invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] + else: + invalidation_mode = None + + success = True + try: + if compile_dests: + for dest in compile_dests: + if os.path.isfile(dest): + if not compile_file(dest, args.ddir, args.force, args.rx, + args.quiet, args.legacy, + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir, + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest): + success = False + else: + if not compile_dir(dest, maxlevels, args.ddir, + args.force, args.rx, args.quiet, + args.legacy, workers=args.workers, + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir, + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest): + success = False + return success + else: + return compile_path(legacy=args.legacy, force=args.force, + quiet=args.quiet, + invalidation_mode=invalidation_mode) + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/macros.pybytecompile b/macros.pybytecompile index d06071e..dff98bc 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -1,7 +1,7 @@ # Note that the path could itself be a python file, or a directory -# Python's compile_all module only works on directories, and requires a max -# recursion depth +# Note that the py_byte_compile macro should work for all Python versions +# Which unfortunately makes the definition more complicated than it should be # Usage: # %py_byte_compile @@ -13,12 +13,22 @@ # (%{py_byte_compile }) || : %py_byte_compile()\ -python_binary="%1"\ -buildroot_path="%2"\ -bytecode_compilation_path=".${buildroot_path/#$RPM_BUILD_ROOT}"\ -failure=0\ -pushd $RPM_BUILD_ROOT\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -m py_compile || failure=1\ -find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -m py_compile || failure=1\ -popd\ -test $failure -eq 0 +py2_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + failure=0\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ + test $failure -eq 0\ +}\ +\ +py3_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ +}\ +\ +# Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ +python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ +# compileall2 Python module is not compatible with Python < 3.4 \ +if [ "$python_version" -ge 34 ]; then py3_byte_compile "%1" "%2"; else py2_byte_compile "%1" "%2"; fi diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 75de388..a6ef5d0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -3,16 +3,18 @@ Version: 3 Release: 46%{?dist} Summary: The unversioned Python RPM macros -License: MIT +# macros: MIT, compileall2.py: PSFv2 +License: MIT and Python Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile +Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py BuildArch: noarch -# For %%python3_pkgversion used in %%python_provide -Requires: python-srpm-macros +# For %%python3_pkgversion used in %%python_provide and compileall2.py +Requires: python-srpm-macros >= 3-46 Obsoletes: python-macros < 3 Provides: python-macros = %{version}-%{release} @@ -25,6 +27,7 @@ python?-devel packages require it. So install a python-devel package instead. %package -n python-srpm-macros Summary: RPM macros for building Python source packages +Requires: redhat-rpm-config %description -n python-srpm-macros RPM macros for building Python source packages. @@ -41,8 +44,6 @@ RPM macros for building Python 2 packages. %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages Requires: python-srpm-macros >= 3-38 -# Would need to be different for each release - worth it? -#Conflicts: python3-devel < 3.5.1-3 %description -n python3-rpm-macros RPM macros for building Python 3 packages. @@ -53,10 +54,13 @@ RPM macros for building Python 3 packages. %build %install -mkdir -p %{buildroot}/%{rpmmacrodir} +mkdir -p %{buildroot}%{rpmmacrodir} install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ - %{buildroot}/%{rpmmacrodir}/ + %{buildroot}%{rpmmacrodir}/ +mkdir -p %{buildroot}%{_rpmconfigdir}/redhat +install -m 644 %{SOURCE5} \ + %{buildroot}%{_rpmconfigdir}/redhat/ %files %{rpmmacrodir}/macros.python @@ -64,6 +68,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm +%{_rpmconfigdir}/redhat/compileall2.py %files -n python2-rpm-macros %{rpmmacrodir}/macros.python2 @@ -76,6 +81,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ * Fri Jul 12 2019 Miro Hrončok - 3-46 - %%python_provide: Switch python2 and python3 behavior - https://fedoraproject.org/wiki/Changes/Python_means_Python3 +- Use compileall2 module for byte-compilation with Python >= 3.4 +- Do not allow passing arguments to Python during byte-compilation +- Use `-s` argument for Python during byte-compilation * Tue Jul 09 2019 Miro Hrončok - 3-45 - %%python_provide: Don't try to obsolete %%_isa provides From 4493789fb8ffead7c09dab8b155f7d1869372d8d Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 11 Jul 2019 14:40:51 +0200 Subject: [PATCH 060/218] Use `-s` to not add user site directory to sys.path for byte-compilation --- macros.pybytecompile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index dff98bc..f36cdbf 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -17,15 +17,15 @@ py2_byte_compile () {\ python_binary="%1"\ bytecode_compilation_path="%2"\ failure=0\ - find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ - find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ + find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -O -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ test $failure -eq 0\ }\ \ py3_byte_compile () {\ python_binary="%1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ }\ \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ From eb3274394cd0738c2e20061242b281b3b05143ba Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 11 Jul 2019 14:42:28 +0200 Subject: [PATCH 061/218] Do not allow passing arguments to Python during byte-compilation --- macros.pybytecompile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/macros.pybytecompile b/macros.pybytecompile index f36cdbf..38ac84a 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -28,6 +28,8 @@ py3_byte_compile () {\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ }\ \ +# Path to intepreter should not contain any arguments \ +[[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 Python module is not compatible with Python < 3.4 \ From e1bd214d259ddbea1545fdeea0be207b7fe38891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Jul 2019 12:15:09 +0200 Subject: [PATCH 062/218] Move brp-python-bytecompile from rpm, so we can easily adapt it --- brp-python-bytecompile | 112 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 brp-python-bytecompile diff --git a/brp-python-bytecompile b/brp-python-bytecompile new file mode 100755 index 0000000..c06bdfa --- /dev/null +++ b/brp-python-bytecompile @@ -0,0 +1,112 @@ +#!/bin/bash +errors_terminate=$2 +extra=$3 + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Figure out how deep we need to descend. We could pick an insanely high +# number and hope it's enough, but somewhere, somebody's sure to run into it. +depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \ + xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c` +if [ -z "$depth" -o "$depth" -le "1" ]; then + exit 0 +fi + +function python_bytecompile() +{ + local options=$1 + local python_binary=$2 + local exclude=$3 + local python_libdir=$4 + local depth=$5 + local real_libdir=$6 + +cat << EOF | $python_binary $options +import compileall, sys, os, re + +python_libdir = "$python_libdir" +depth = $depth +real_libdir = "$real_libdir" +build_root = "$RPM_BUILD_ROOT" +exclude = r"$exclude" + +class Filter: + def search(self, path): + ret = not os.path.realpath(path).startswith(build_root) + if exclude: + ret = ret or re.search(exclude, path) + return ret + +sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) +EOF +} + +# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python +# bytecode that they are for. +# +# The files below RPM_BUILD_ROOT could be targeting multiple versions of +# python (e.g. a single build that emits several subpackages e.g. a +# python26-foo subpackage, a python31-foo subpackage etc) +# +# Support this by assuming that below each /usr/lib/python$VERSION/, all +# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. +# +# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 +# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 + +shopt -s nullglob +for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]$"`; +do + python_binary=/usr/bin/$(basename $python_libdir) + real_libdir=${python_libdir/$RPM_BUILD_ROOT/} + echo "Bytecompiling .py files below $python_libdir using $python_binary" + + # Generate normal (.pyc) byte-compiled files. + python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi + + # Generate optimized (.pyo) byte-compiled files. + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 + fi +done + + +# Handle other locations in the filesystem using the default python implementation +# if extra is set to 0, don't do this +if [ 0$extra -eq 0 ]; then + exit 0 +fi + +# If we don't have a default python interpreter, we cannot proceed +default_python=${1:-/usr/bin/python} +if [ ! -x "$default_python" ]; then + exit 0 +fi + +# Figure out if there are files to be bytecompiled with the default_python at all +# this prevents unnecessary default_python invocation +find "$RPM_BUILD_ROOT" -type f -name "*.py" | grep -Ev "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" || exit 0 + +# Generate normal (.pyc) byte-compiled files. +python_bytecompile "" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" +if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 +fi + +# Generate optimized (.pyo) byte-compiled files. +python_bytecompile "-O" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" +if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had a syntax error + exit 1 +fi +exit 0 From e64ffd7f264d4172edbefad79535d7c966f36306 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Sat, 20 Jul 2019 08:08:26 +0200 Subject: [PATCH 063/218] Use compileall2 Python module for byte-compilation in brp-python-bytecompile --- brp-python-bytecompile | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index c06bdfa..fee0048 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -15,6 +15,14 @@ if [ -z "$depth" -o "$depth" -le "1" ]; then exit 0 fi +# This function now implements Python byte-compilation in two different ways: +# Python >= 3.4 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks +# When we drop support for Python 2, we'd be able to use all compileall2 features like: +# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir +# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options +# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() +# These changes will make this script much simpler function python_bytecompile() { local options=$1 @@ -24,6 +32,26 @@ function python_bytecompile() local depth=$5 local real_libdir=$6 + python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + + # + # Python 3.4 and higher + # + if [ "$python_version" -ge 34 ]; then + + [ ! -z $exclude ] && exclude="-x '$exclude'" + # /usr/lib/rpm/redhat/ contains compileall2 Python module + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary $options -m compileall2 -q -f $exclude -d $real_libdir -e $RPM_BUILD_ROOT $python_libdir + else +# +# Python 3.3 and lower (incl. Python 2) +# + cat << EOF | $python_binary $options import compileall, sys, os, re @@ -42,6 +70,8 @@ class Filter: sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) EOF + +fi } # .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python From 2cbca3f95ea79a9ac72d124d4e97d62f891babca Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 26 Jul 2019 16:06:16 +0000 Subject: [PATCH 064/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a6ef5d0..b1eb6d0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 46%{?dist} +Release: 47%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Fri Jul 26 2019 Fedora Release Engineering - 3-47 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild + * Fri Jul 12 2019 Miro Hrončok - 3-46 - %%python_provide: Switch python2 and python3 behavior - https://fedoraproject.org/wiki/Changes/Python_means_Python3 From 84ed1ab69d4b1e0aea452350d9f2865e8f7c28e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 26 Aug 2019 15:23:42 +0200 Subject: [PATCH 065/218] Fix %python3_version macros for Python 3.10 No need to bump the release, 3.10 is far from now. Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1745601 --- macros.python3 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros.python3 b/macros.python3 index 1952aed..a170c4b 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,7 +1,7 @@ %python3_sitelib %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3])") -%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write(sys.version[:3].replace('.',''))") +%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} From f09ccd21f57eb962cbb3fcee33190c0dfdbc6f8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 26 Aug 2019 09:34:39 +0200 Subject: [PATCH 066/218] Drop --strip-file-prefix option from %pyX_install_wheel macros, it is not needed A custom pip patch was needed for this option, but the RECORD files are relative, so no stripping is needed. We will eventually drop the patch. --- macros.python2 | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/macros.python2 b/macros.python2 index 38c6eb9..4ff5f41 100644 --- a/macros.python2 +++ b/macros.python2 @@ -41,5 +41,5 @@ } %py2_install_wheel() %{expand:\\\ - pip%{python2_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps + pip%{python2_version} install -I dist/%{1} --root %{buildroot} --no-deps } diff --git a/macros.python3 b/macros.python3 index a170c4b..20d6700 100644 --- a/macros.python3 +++ b/macros.python3 @@ -35,5 +35,5 @@ } %py3_install_wheel() %{expand:\\\ - pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps + pip%{python3_version} install -I dist/%{1} --root %{buildroot} --no-deps } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b1eb6d0..1355a69 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 47%{?dist} +Release: 48%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Aug 26 2019 Miro Hrončok - 3-48 +- Drop --strip-file-prefix option from %%pyX_install_wheel macros, it is not needed + * Fri Jul 26 2019 Fedora Release Engineering - 3-47 - Rebuilt for https://fedoraproject.org/wiki/Fedora_31_Mass_Rebuild From af35bb0ac96570d4842bee16a1f8432907a8ab18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 1 Aug 2019 13:48:51 +0200 Subject: [PATCH 067/218] Define %python2 and %python3 See https://pagure.io/packaging-committee/issue/907 Redefine %__pythonX to change the behavior of %pythonX, %pythonX_version, etc. Use %pythonX in spec. --- macros.python-srpm | 6 ++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index fb960ea..78bde41 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -2,9 +2,15 @@ # - they can be used in Build/Requires # - they can be used in non-Python packages where requiring pythonX-devel would # be an overkill + +# use the underscored macros to redefine the behavior of %%python3_version etc. %__python2 /usr/bin/python2 %__python3 /usr/bin/python3 +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 +%python3 %__python3 + # For backwards compatibility only # See the comments in https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/22 %__python /usr/bin/python diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 1355a69..4dfdf20 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 48%{?dist} +Release: 49%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Fri Sep 27 2019 Miro Hrončok - 3-49 +- Define %%python2 and %%python3 + * Mon Aug 26 2019 Miro Hrončok - 3-48 - Drop --strip-file-prefix option from %%pyX_install_wheel macros, it is not needed From dee3401a7efa5f30143526e56e222d7fdcaedac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 30 Sep 2019 14:35:43 +0200 Subject: [PATCH 068/218] Define %python2 and %python3 See https://pagure.io/packaging-committee/issue/907 Redefine %__pythonX to change the behavior of %pythonX, %pythonX_version, etc. Use %pythonX in spec. --- macros.python2 | 5 +++++ macros.python3 | 6 ++++++ python-rpm-macros.spec | 5 ++++- 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/macros.python2 b/macros.python2 index 66e9c5a..0a75d76 100644 --- a/macros.python2 +++ b/macros.python2 @@ -1,4 +1,9 @@ +# use the underscored macros to redefine the behavior of %%python2_version etc. %__python2 /usr/bin/python2.6 + +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 + %python2_sitelib %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python2_sitearch %(%{__python2} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python2_version %(%{__python2} -c "import sys; sys.stdout.write(sys.version[:3])") diff --git a/macros.python3 b/macros.python3 index 7226ccc..41345e3 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,5 +1,11 @@ +# use the underscored macros to redefine the behavior of %%python3_version etc. %__python3 /usr/bin/python3.4 + +# use the non-underscored macros to refer to Python in spec, etc. +%python3 %__python3 + #__python3_other /usr/bin/python3.5 +#python3_other %%__python3_other %python3_sitelib %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib())") %python3_sitearch %(%{__python3} -c "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c4ad781..0f73e16 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 14%{?dist} +Release: 15%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -65,6 +65,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE5} \ %changelog +* Fri Sep 27 2019 Miro Hrončok - 3-15 +- Define %%python2 and %%python3 + * Wed Mar 06 2019 Carl George - 3-14 - Move macros.pybytecompile in here from python3X-devel - macros.pybytecompile: Detect Python version through sys.version_info instead From d4418433fa1c204527a3a49db11f89ac9f6ab657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 2 Oct 2019 16:20:57 +0200 Subject: [PATCH 069/218] Drop hardcoded python2 from %py_build --- macros.python | 2 +- python-rpm-macros.spec | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 94e769c..2153629 100644 --- a/macros.python +++ b/macros.python @@ -2,7 +2,7 @@ %py_shbang_opts -s %py_build() %{expand:\ -CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py_shbang_opts}" %{?1}\ +CFLAGS="%{optflags}" %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?1}\ } %py_install() %{expand:\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0f73e16..0d33824 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -67,6 +67,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE5} \ %changelog * Fri Sep 27 2019 Miro Hrončok - 3-15 - Define %%python2 and %%python3 +- Drop hardcoded python2 from %%py_build * Wed Mar 06 2019 Carl George - 3-14 - Move macros.pybytecompile in here from python3X-devel From bebf85d28b8cfb14b8f2307efb45d18b22ef23bd Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 26 Nov 2019 14:04:57 +0100 Subject: [PATCH 070/218] Bundled compileall2 module update to 0.6.0 --- compileall2.py | 34 ++++++++++++++++++++++++---------- python-rpm-macros.spec | 5 ++++- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/compileall2.py b/compileall2.py index b2c1e97..593dda9 100644 --- a/compileall2.py +++ b/compileall2.py @@ -9,6 +9,10 @@ recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) See module py_compile for details of the actual byte-compilation. + +License: +Compileall2 is an enhanced copy of Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ """ import os import sys @@ -39,8 +43,6 @@ else: pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) -RECURSION_LIMIT = sys.getrecursionlimit() - __all__ = ["compile_dir","compile_file","compile_path"] def optimization_kwarg(opt): @@ -56,7 +58,7 @@ def optimization_kwarg(opt): else: return dict() -def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): +def _walk_dir(dir, maxlevels, quiet=0): if PY36 and quiet < 2 and isinstance(dir, os.PathLike): dir = os.fspath(dir) else: @@ -81,7 +83,7 @@ def _walk_dir(dir, maxlevels=RECURSION_LIMIT, quiet=0): yield from _walk_dir(fullname, maxlevels=maxlevels - 1, quiet=quiet) -def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, +def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, prependdir=None, limit_sl_dest=None): @@ -119,6 +121,8 @@ def compile_dir(dir, maxlevels=RECURSION_LIMIT, ddir=None, force=False, from concurrent.futures import ProcessPoolExecutor except ImportError: workers = 1 + if maxlevels is None: + maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True if workers is not None and workers != 1 and ProcessPoolExecutor is not None: @@ -169,6 +173,11 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. """ + + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + success = True if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): fullname = os.fspath(fullname) @@ -323,7 +332,7 @@ def main(): parser = argparse.ArgumentParser( description='Utilities to support installing Python libraries.') parser.add_argument('-l', action='store_const', const=0, - default=RECURSION_LIMIT, dest='maxlevels', + default=None, dest='maxlevels', help="don't recurse into subdirectories") parser.add_argument('-r', type=int, dest='recursion', help=('control the maximum recursion level. ' @@ -345,16 +354,16 @@ def main(): default=None, help=('part of path to left-strip from path ' 'to source file - for example buildroot. ' - 'if `-d` and `-s` options are specified, ' - 'then `-d` takes precedence.')) + '`-d` and `-s` options cannot be ' + 'specified together.')) parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir', default=None, help=('path to add as prefix to path ' 'to source file - for example / to make ' 'it absolute when some part is removed ' - 'by `-s` option' - 'if `-d` and `-a` options are specified, ' - 'then `-d` takes precedence.')) + 'by `-s` option. ' + '`-d` and `-p` options cannot be ' + 'specified together.')) parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, help=('skip files matching the regular expression; ' 'the regexp is searched for in the full path ' @@ -404,6 +413,11 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if args.ddir is not None and ( + args.stripdir is not None or args.prependdir is not None + ): + parser.error("-d cannot be used in combination with -s or -p") + # if flist is provided then load it if args.flist: try: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4dfdf20..2ad43ea 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 49%{?dist} +Release: 50%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Tue Nov 26 2019 Lumír Balhar - 3-50 +- Update of bundled compileall2 module + * Fri Sep 27 2019 Miro Hrončok - 3-49 - Define %%python2 and %%python3 From 8f6bc2fd6ca40fc74f67f914f4109ed54ee63c5b Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 5 Nov 2019 09:40:12 +0100 Subject: [PATCH 071/218] Fix brp-python-bytecompile with the new features from compileall2 Resolves: rhbz#1595265 The problem this change is intended to solve is with how `real_libdir` is calculated. Let's assume we want to recursively byte-compile all `*.py` files in `/builddir/build/BUILDROOT/python-scales-1.0.9-250.fc32.x86_64/usr/lib/python3.8`. Then, `real_libdir` is this path without `$RPM_BUILD_ROOT` with the filename at the end which displays in the error message like this: ``` Bytecompiling .py files below /builddir/build/BUILDROOT/python-scales-1.0.9-250.fc32.x86_64/usr/lib/python3.8 using /usr/bin/python3.8 *** Error compiling '/builddir/build/BUILDROOT/python-scales-1.0.9-250.fc32.x86_64/usr/lib/python3.8/site-packages/greplin/bar.py'... File "/usr/lib/python3.8/bar.py", line 1 import sin from math ^ SyntaxError: invalid syntax ``` `/usr/lib/python3.8/bar.py` is obviously wrong. One of the new features of the `compileall2` module (which will be available in stdlib in Python 3.9) is that the path byte-compiled to `*.pyc` files is calculated for each file. This means that by using `-s` and `-p` we can strip `$RPM_BUILD_ROOT` and prepend `/` for each file individually which will fix the problem. ``` Bytecompiling .py files below /builddir/build/BUILDROOT/python-scales-1.0.9-250.fc32.x86_64/usr/lib/python3.8 using /usr/bin/python3.8 *** Error compiling '/builddir/build/BUILDROOT/python-scales-1.0.9-250.fc32.x86_64/usr/lib/python3.8/site-packages/greplin/bar.py'... File "/usr/lib/python3.8/site-packages/greplin/bar.py", line 1 import sin from math ^ SyntaxError: invalid syntax ``` This change has an effect only for Python >= 3.4. --- brp-python-bytecompile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index fee0048..ede435b 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -29,8 +29,8 @@ function python_bytecompile() local python_binary=$2 local exclude=$3 local python_libdir=$4 - local depth=$5 - local real_libdir=$6 + local depth=$5 # Not used for Python >= 3.4 + local real_libdir=$6 # Not used for Python >= 3.4 python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") @@ -46,7 +46,9 @@ function python_bytecompile() # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root # -x and -e together implements the same functionality as the Filter class below - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary $options -m compileall2 -q -f $exclude -d $real_libdir -e $RPM_BUILD_ROOT $python_libdir + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary $options -m compileall2 -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir else # # Python 3.3 and lower (incl. Python 2) From f0be0a29835353067e4d31f2caee5b11cfa1931a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 28 Dec 2019 19:10:09 +0100 Subject: [PATCH 072/218] Define %python, but make it work only if %__python is redefined --- macros.python-srpm | 9 +++++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 78bde41..9fa4f06 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -15,6 +15,15 @@ # See the comments in https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/22 %__python /usr/bin/python +# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) +%python() %{lua:\ + __python = rpm.expand("%__python")\ + if __python == "/usr/bin/python" then\ + rpm.expand("%{error:Cannot use %%python if %%__python wasn't redefined to something other than /usr/bin/python.}")\ + else\ + print(__python)\ + end\ +} # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 2ad43ea..3506e56 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 50%{?dist} +Release: 51%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Sat Dec 28 2019 Miro Hrončok - 3-51 +- Define %%python, but make it work only if %%__python is redefined + * Tue Nov 26 2019 Lumír Balhar - 3-50 - Update of bundled compileall2 module From 6c63a5b7f484f3c4b6bee13196561c9b1dff410a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 28 Dec 2019 19:12:37 +0100 Subject: [PATCH 073/218] Add the %pycached macro Usage: %files ... %pycached %{python3_sitelib}/foo.py This will list: /usr/lib/python3.8/site-packages/foo.py /usr/lib/python3.8/site-packages/__pycache__/foo.cpython-38{,.opt-?}.pyc Assuming the Python 3 version is 3.8. The bytecode files are globbed, their presence is not checked. This will fail: %pycached %{python3_sitelib}/foo error: %pycached can only be used with paths explicitly ending with .py And so will any of this: %pycached %{python3_sitelib}/* %pycached %{python3_sitelib}/foo.* %pycached %{python3_sitelib}/foo.p? %pycached %{python3_sitelib}/foo.?y %pycached %{python3_sitelib}/foo.?? But this will work: %pycached %{python3_sitelib}/foo*.py And it will generate the following globs: /usr/lib/python3.8/site-packages/foo*.py /usr/lib/python3.8/site-packages/__pycache__/foo*.cpython-38{,.opt-?}.pyc When used with paths that include Python 3 version, it globs with the version: %pycached /opt/python3.10/foo.py Generates: /opt/python3.10/foo.py /opt/python3.10/__pycache__/foo.cpython-310{,.opt-?}.pyc While paths without version have less strict globs: %pycached /custom/foo.py /custom/foo.py /custom/__pycache__/foo.cpython-3*{,.opt-?}.pyc This will generate a warning in RPM build: warning: File listed twice: /custom/__pycache__/foo.cpython-38.opt-1.pyc However it ensures the optimized bytecode is there. --- macros.python3 | 15 +++++++++++++++ python-rpm-macros.spec | 1 + 2 files changed, 16 insertions(+) diff --git a/macros.python3 b/macros.python3 index 20d6700..b772fb4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -37,3 +37,18 @@ %py3_install_wheel() %{expand:\\\ pip%{python3_version} install -I dist/%{1} --root %{buildroot} --no-deps } + +# This only supports Python 3.5+ and will never work with Python 2. +# Hence, it has no Python version in the name. +%pycached() %{lua: + path = rpm.expand("%{?1}") + if (string.sub(path, "-3") ~= ".py") then + rpm.expand("%{error:%%pycached can only be used with paths explicitly ending with .py}") + else + print(path) + pyminor = path:match("/python3.(%d+)/") or "*" + dirname = path:match("(.*/)") + modulename = path:match(".*/([^/]+).py") + print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") + end +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3506e56..79ed93b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -80,6 +80,7 @@ install -m 644 %{SOURCE5} \ %changelog * Sat Dec 28 2019 Miro Hrončok - 3-51 - Define %%python, but make it work only if %%__python is redefined +- Add the %%pycached macro * Tue Nov 26 2019 Lumír Balhar - 3-50 - Update of bundled compileall2 module From 2314fd928a6940dccb1ffd3ea799e3a779df9645 Mon Sep 17 00:00:00 2001 From: Anna Khaitovich Date: Sat, 28 Dec 2019 19:16:16 +0100 Subject: [PATCH 074/218] Remove stray __pycache__ directory from /usr/bin when running %py_install, %py_install_wheel and %py_install_egg macros Solves bz#1739848 --- macros.python | 3 +++ macros.python2 | 3 +++ macros.python3 | 3 +++ python-rpm-macros.spec | 2 ++ 4 files changed, 11 insertions(+) diff --git a/macros.python b/macros.python index e431b92..575ddcb 100644 --- a/macros.python +++ b/macros.python @@ -31,15 +31,18 @@ %py_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python_sitelib} easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py_install_wheel() %{expand:\\\ pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %python_provide() %{lua: diff --git a/macros.python2 b/macros.python2 index 4ff5f41..16e8e4b 100644 --- a/macros.python2 +++ b/macros.python2 @@ -33,13 +33,16 @@ %py2_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py2_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python2_sitelib} easy_install-%{python2_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py2_install_wheel() %{expand:\\\ pip%{python2_version} install -I dist/%{1} --root %{buildroot} --no-deps + rm -rfv %{buildroot}%{_bindir}/__pycache__ } diff --git a/macros.python3 b/macros.python3 index b772fb4..36c156e 100644 --- a/macros.python3 +++ b/macros.python3 @@ -27,15 +27,18 @@ %py3_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py3_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python3_sitelib} easy_install-%{python3_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py3_install_wheel() %{expand:\\\ pip%{python3_version} install -I dist/%{1} --root %{buildroot} --no-deps + rm -rfv %{buildroot}%{_bindir}/__pycache__ } # This only supports Python 3.5+ and will never work with Python 2. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 79ed93b..c060871 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -81,6 +81,8 @@ install -m 644 %{SOURCE5} \ * Sat Dec 28 2019 Miro Hrončok - 3-51 - Define %%python, but make it work only if %%__python is redefined - Add the %%pycached macro +- Remove stray __pycache__ directory from /usr/bin when running %%py_install, + %%py_install_wheel and %%py_build_wheel macros * Tue Nov 26 2019 Lumír Balhar - 3-50 - Update of bundled compileall2 module From fa7d708e3c06e24ce7de8ea49d69be5a2615df9c Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 22 Jan 2020 13:09:31 +0100 Subject: [PATCH 075/218] Use `-B` flag for Python when using compileall2 to not write pyc files The Python compileall2 module in /usr/lib/rpm/redhat/ can be executed by various different Python interpreters. We don't want to write several different `*.pyc` files to this location - in most cases, that's not possible, but somebody might run this as root. --- brp-python-bytecompile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index ede435b..ed1563b 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -48,7 +48,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary $options -m compileall2 -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir else # # Python 3.3 and lower (incl. Python 2) From 82f62228c96def55dba00a21d93e8702dd149107 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Thu, 30 Jan 2020 15:05:59 +0000 Subject: [PATCH 076/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c060871..322d854 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 51%{?dist} +Release: 52%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Thu Jan 30 2020 Fedora Release Engineering - 3-52 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild + * Sat Dec 28 2019 Miro Hrončok - 3-51 - Define %%python, but make it work only if %%__python is redefined - Add the %%pycached macro From e9f07b72aa3c64ff2f5b371c5ea0651b933860a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 6 Feb 2020 10:24:03 +0100 Subject: [PATCH 077/218] Define %py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a --- macros.python | 1 + macros.python2 | 1 + macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 575ddcb..fe30d71 100644 --- a/macros.python +++ b/macros.python @@ -7,6 +7,7 @@ %py_setup setup.py %py_shbang_opts -s +%py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 16e8e4b..e004554 100644 --- a/macros.python2 +++ b/macros.python2 @@ -4,6 +4,7 @@ %python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %py2_shbang_opts -s +%py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 36c156e..f3727b7 100644 --- a/macros.python3 +++ b/macros.python3 @@ -6,6 +6,7 @@ %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s +%py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 322d854..c6fe8c6 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 52%{?dist} +Release: 53%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Thu Feb 06 2020 Miro Hrončok - 3-53 +- Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a + * Thu Jan 30 2020 Fedora Release Engineering - 3-52 - Rebuilt for https://fedoraproject.org/wiki/Fedora_32_Mass_Rebuild From 456f3ecffb062d1594285b6f9aec0ef2f9f22391 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 10 Feb 2020 23:47:52 +0100 Subject: [PATCH 078/218] Update of bundled compileall2 module to 0.7.0 Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication This is explained in https://discuss.python.org/t/3014 and https://github.com/fedora-python/compileall2/issues/16 This option is not yet used anywhere. That allows us to backport this to all Fedoras but only use --hardlink-dupes on rawhide first. --- compileall2.py | 46 ++++++++++++++++++++++++++++++++++-------- python-rpm-macros.spec | 8 ++++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/compileall2.py b/compileall2.py index 593dda9..8976fa0 100644 --- a/compileall2.py +++ b/compileall2.py @@ -19,6 +19,7 @@ import sys import importlib.util import py_compile import struct +import filecmp from functools import partial from pathlib import Path @@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0): def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, - prependdir=None, limit_sl_dest=None): + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path + hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None if workers is not None: @@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest): + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, stripdir=None, prependdir=None, - limit_sl_dest=None): + limit_sl_dest=None, hardlink_dupes=False): """Byte-compile one file. Arguments (only fullname is required): @@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. + hardlink_dupes: hardlink duplicated pyc files """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if isinstance(optimize, int): optimize = [optimize] + if hardlink_dupes: + raise ValueError(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if rx is not None: mo = rx.search(fullname) if mo: @@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not quiet: print('Compiling {!r}...'.format(fullname)) try: - for opt_level, cfile in opt_cfiles.items(): + for index, opt_level in enumerate(sorted(optimize)): + cfile = opt_cfiles[opt_level] if PY37: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level, @@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level) + + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if previous_cfile == cfile and optimize[0] not in (1, 2): + # Python 3.4 has only one .pyo file for -O and -OO so + # we hardlink it only if there is a .pyc file + # with the same content + previous_cfile = opt_cfiles[optimize[0]] + if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) + except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -379,11 +400,14 @@ def main(): parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', - help=('Optimization levels to run compilation with.' - 'Default is -1 which uses optimization level of' + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses optimization level of ' 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') if PY37: invalidation_modes = [mode.name.lower().replace('_', '-') @@ -413,6 +437,10 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if args.ddir is not None and ( args.stripdir is not None or args.prependdir is not None ): @@ -449,7 +477,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -459,7 +488,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False return success else: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c6fe8c6..0734fe4 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 53%{?dist} +Release: 54%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Feb 10 2020 Miro Hrončok - 3-54 +- Update of bundled compileall2 module to 0.7.0 + Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication + * Thu Feb 06 2020 Miro Hrončok - 3-53 - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a From 229fd899ac20d25a66efb9c09841d0b0c5ece7c9 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Mon, 27 Jan 2020 13:29:41 +0100 Subject: [PATCH 079/218] brp-python-bytecompile: Prepare for 2 digit minor versions (e.g. 3.10) --- brp-python-bytecompile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index ed1563b..4727feb 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -90,7 +90,7 @@ fi # and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 shopt -s nullglob -for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]$"`; +for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]+$"`; do python_binary=/usr/bin/$(basename $python_libdir) real_libdir=${python_libdir/$RPM_BUILD_ROOT/} From 1b3e731dc657cc392d4e9b7b16725ad5aaa67360 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Mar 2020 12:17:38 +0100 Subject: [PATCH 080/218] Brand as "3.8", rework %python_provide - Hardcode the default Python 3 version in the SRPM macros - Provide python38-foo for python3-foo and the other way around (future RHEL compatibility) Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1812087 $ rpm --eval '%python_provide python38-setuptools' Provides: python-setuptools = %{version}-%{release} Provides: python3-setuptools = %{version}-%{release} Obsoletes: python-setuptools < %{version}-%{release} $ rpm --eval '%python_provide python3-setuptools' Provides: python-setuptools = %{version}-%{release} Provides: python38-setuptools = %{version}-%{release} Obsoletes: python-setuptools < %{version}-%{release} $ rpm --eval '%python_provide python39-setuptools' $ rpm --define 'python3_pkgversion 39' --eval '%python_provide python%{python3_pkgversion}-setuptools' To make the implementation of %python_provide easier, any names starting with "python" or "pypy" are recognized as valid arguments. Previously, this was an ERROR: $ rpm --eval '%python_provide pythonista' %python_provide: ERROR: pythonista not recognized. Now it is a no-op. The behavior was never documented and the change is backwards compatible for working spec files. --- macros.python | 40 ++++++++++++++++++++++++++-------------- macros.python-srpm | 5 +++++ python-rpm-macros.spec | 11 ++++++++--- 3 files changed, 39 insertions(+), 17 deletions(-) diff --git a/macros.python b/macros.python index fe30d71..0b00946 100644 --- a/macros.python +++ b/macros.python @@ -52,11 +52,29 @@ end package = rpm.expand("%{?1}") vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") - if (string.starts(package, "python2-")) then - --No unversioned provides as python2 is not default - elseif (string.starts(package, "python" .. rpm.expand("%{python3_pkgversion}") .. "-")) then + if (string.starts(package, "python3-")) then print("\\nProvides: python-") - l = 8 + string.len(rpm.expand("%{python3_pkgversion}")) + print(string.sub(package,9,string.len(package))) + print(" = ") + print(vr) + print("\\nProvides: python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-") + print(string.sub(package,9,string.len(package))) + print(" = ") + print(vr) + --Obsoleting the previous default python package (if it doesn't have isa) + if (string.sub(package, "-1") ~= ")") then + print("\\nObsoletes: python-") + print(string.sub(package,9,string.len(package))) + print(" < ") + print(vr) + end + elseif (string.starts(package, "python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-")) then + print("\\nProvides: python-") + l = 8 + string.len(rpm.expand("%{__default_python3_pkgversion}")) + print(string.sub(package,l,string.len(package))) + print(" = ") + print(vr) + print("\\nProvides: python3-") print(string.sub(package,l,string.len(package))) print(" = ") print(vr) @@ -67,16 +85,10 @@ print(" < ") print(vr) end - elseif (rpm.expand("%{?python3_other_pkgversion}") ~= "" and string.starts(package, "python" .. rpm.expand("%{python3_other_pkgversion}") .. "-")) then - --No unversioned provides as python3_other is not default - elseif (string.starts(package, "pypy-")) then - --No unversioned provides as pypy is not default - elseif (string.starts(package, "pypy2-")) then - --No unversioned provides as pypy is not default - elseif (string.starts(package, "pypy3-")) then - --No unversioned provides as pypy is not default - elseif (string.starts(package, "python-")) then - --No unversioned provides needed for unversioned python + elseif (string.starts(package, "python")) then + --No unversioned provides as other python3 cases are not the default + elseif (string.starts(package, "pypy")) then + --No unversioned provides as pypy is not default either else print("%python_provide: ERROR: ") print(package) diff --git a/macros.python-srpm b/macros.python-srpm index 9fa4f06..67e53c9 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -25,6 +25,11 @@ end\ } +# This specifies what packages are equal to the python3-... packages, in python_provide macro +# E.g. in Fedora 32, python38-foo will provide python3-foo and vice versa +%__default_python3_version 3.8 +%__default_python3_pkgversion %(echo %__default_python3_version | sed 's/\\.//') + # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0734fe4..dd5ea6a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,7 +1,7 @@ Name: python-rpm-macros -Version: 3 -Release: 54%{?dist} -Summary: The unversioned Python RPM macros +Version: 3.8 +Release: 1%{?dist} +Summary: The common Python RPM macros # macros: MIT, compileall2.py: PSFv2 License: MIT and Python @@ -78,6 +78,11 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Mar 23 2020 Miro Hrončok - 3.8-1 +- Hardcode the default Python 3 version in the SRPM macros (#1812087) +- Provide python38-foo for python3-foo and the other way around (future RHEL compatibility) +- %%python_provide: Allow any names starting with "python" or "pypy" + * Mon Feb 10 2020 Miro Hrončok - 3-54 - Update of bundled compileall2 module to 0.7.0 Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication From 15e32a300599a69f29d060abe989c4b39f954060 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 3 Mar 2020 15:26:45 +0100 Subject: [PATCH 081/218] Update of bundled compileall2 module to 0.7.1 (bugfix release) --- compileall2.py | 7 +++++++ python-rpm-macros.spec | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compileall2.py b/compileall2.py index 8976fa0..c58e545 100644 --- a/compileall2.py +++ b/compileall2.py @@ -113,6 +113,13 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + if ddir is not None: + stripdir = dir + prependdir = ddir + ddir = None if workers is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index dd5ea6a..e10be21 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 1%{?dist} +Release: 2%{?dist} Summary: The common Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Tue Mar 31 2020 Lumír Balhar - 3.8-2 +- Update of bundled compileall2 module to 0.7.1 (bugfix release) + * Mon Mar 23 2020 Miro Hrončok - 3.8-1 - Hardcode the default Python 3 version in the SRPM macros (#1812087) - Provide python38-foo for python3-foo and the other way around (future RHEL compatibility) From fed99a047898c3f62dc6ffa1f9edb9f06511ca94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 27 Apr 2020 10:40:14 +0200 Subject: [PATCH 082/218] Make pythonX-rpm-macros depend on python-rpm-macros %pyX_(build|install) uses %py_setup from python-rpm-macros Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1827811 --- python-rpm-macros.spec | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e10be21..134e4e5 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 2%{?dist} +Release: 3%{?dist} Summary: The common Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -35,6 +35,7 @@ RPM macros for building Python source packages. %package -n python2-rpm-macros Summary: RPM macros for building Python 2 packages Requires: python-srpm-macros >= 3-38 +Requires: python-rpm-macros # Would need to be different for each release - worth it? #Conflicts: python2-devel < 2.7.11-3 @@ -44,6 +45,7 @@ RPM macros for building Python 2 packages. %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages Requires: python-srpm-macros >= 3-38 +Requires: python-rpm-macros %description -n python3-rpm-macros RPM macros for building Python 3 packages. @@ -78,6 +80,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Apr 27 2020 Miro Hrončok - 3.8-3 +- Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) + * Tue Mar 31 2020 Lumír Balhar - 3.8-2 - Update of bundled compileall2 module to 0.7.1 (bugfix release) From daf7d326123a0f8c042a1a5398da88d7eaa9bec0 Mon Sep 17 00:00:00 2001 From: Tomas Hrnciar Date: Mon, 4 May 2020 11:01:42 +0200 Subject: [PATCH 083/218] remove direct_url.json file and sed it out from RECORD With PEP 610 there is created new file direct_url.json since is not useful for us, it will be removed using py3_install_wheel macro. See: https://discuss.python.org/t/pep-610-usage-guidelines-for-linux-distributions/4012 --- macros.python | 6 ++++++ macros.python2 | 6 ++++++ macros.python3 | 6 ++++++ python-rpm-macros.spec | 6 +++++- 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 0b00946..d69dd4f 100644 --- a/macros.python +++ b/macros.python @@ -44,6 +44,12 @@ %py_install_wheel() %{expand:\\\ pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ + for distinfo in %{buildroot}%{python_sitelib}/*.dist-info %{buildroot}%{python_sitearch}/*.dist-info; do + if [ -f ${distinfo}/direct_url.json ]; then + rm -fv ${distinfo}/direct_url.json + sed -i '/direct_url.json/d' ${distinfo}/RECORD + fi + done } %python_provide() %{lua: diff --git a/macros.python2 b/macros.python2 index e004554..82d7428 100644 --- a/macros.python2 +++ b/macros.python2 @@ -46,4 +46,10 @@ %py2_install_wheel() %{expand:\\\ pip%{python2_version} install -I dist/%{1} --root %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ + for distinfo in %{buildroot}%{python2_sitelib}/*.dist-info %{buildroot}%{python2_sitearch}/*.dist-info; do + if [ -f ${distinfo}/direct_url.json ]; then + rm -fv ${distinfo}/direct_url.json + sed -i '/direct_url.json/d' ${distinfo}/RECORD + fi + done } diff --git a/macros.python3 b/macros.python3 index f3727b7..7af56f1 100644 --- a/macros.python3 +++ b/macros.python3 @@ -40,6 +40,12 @@ %py3_install_wheel() %{expand:\\\ pip%{python3_version} install -I dist/%{1} --root %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ + for distinfo in %{buildroot}%{python3_sitelib}/*.dist-info %{buildroot}%{python3_sitearch}/*.dist-info; do + if [ -f ${distinfo}/direct_url.json ]; then + rm -fv ${distinfo}/direct_url.json + sed -i '/direct_url.json/d' ${distinfo}/RECORD + fi + done } # This only supports Python 3.5+ and will never work with Python 2. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 134e4e5..c975feb 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 3%{?dist} +Release: 4%{?dist} Summary: The common Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -80,6 +80,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon May 04 2020 Tomas Hrnciar - 3.8-4 +- Make %%py3_install_wheel macro remove direct_url.json file created by PEP 610. +- https://discuss.python.org/t/pep-610-usage-guidelines-for-linux-distributions/4012 + * Mon Apr 27 2020 Miro Hrončok - 3.8-3 - Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) From b314efc5a74bf898139d1029db4e9a63a50fc652 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 15:31:23 +0200 Subject: [PATCH 084/218] Add common Lua functions, use a Lua function in %python_provide --- macros.python | 37 +++++++++++++------------------------ python-rpm-macros.spec | 7 ++++++- python.lua | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 25 deletions(-) create mode 100644 python.lua diff --git a/macros.python b/macros.python index d69dd4f..0044fae 100644 --- a/macros.python +++ b/macros.python @@ -53,43 +53,32 @@ } %python_provide() %{lua: + local python = require "fedora.srpm.python" function string.starts(String,Start) return string.sub(String,1,string.len(Start))==Start end - package = rpm.expand("%{?1}") - vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") + local package = rpm.expand("%{?1}") + local vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") + local altnames = python.python_altnames(package) if (string.starts(package, "python3-")) then - print("\\nProvides: python-") - print(string.sub(package,9,string.len(package))) - print(" = ") - print(vr) - print("\\nProvides: python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-") - print(string.sub(package,9,string.len(package))) - print(" = ") - print(vr) + for i, altname in ipairs(altnames) do + print("\\nProvides: " .. altname .. " = " .. vr) + end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") print(string.sub(package,9,string.len(package))) - print(" < ") - print(vr) + print(" < " .. vr) end elseif (string.starts(package, "python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-")) then - print("\\nProvides: python-") - l = 8 + string.len(rpm.expand("%{__default_python3_pkgversion}")) - print(string.sub(package,l,string.len(package))) - print(" = ") - print(vr) - print("\\nProvides: python3-") - print(string.sub(package,l,string.len(package))) - print(" = ") - print(vr) + for i, altname in ipairs(altnames) do + print("\\nProvides: " .. altname .. " = " .. vr) + end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") - print(string.sub(package,l,string.len(package))) - print(" < ") - print(vr) + print(string.sub(package,10,string.len(package))) + print(" < " .. vr) end elseif (string.starts(package, "python")) then --No unversioned provides as other python3 cases are not the default diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c975feb..4f635f3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -3,7 +3,7 @@ Version: 3.8 Release: 4%{?dist} Summary: The common Python RPM macros -# macros: MIT, compileall2.py: PSFv2 +# macros and lua: MIT, compileall2.py: PSFv2 License: MIT and Python Source0: macros.python Source1: macros.python-srpm @@ -11,6 +11,7 @@ Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile Source5: https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py +Source6: python.lua BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -64,6 +65,9 @@ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 %{SOURCE5} \ %{buildroot}%{_rpmconfigdir}/redhat/ +mkdir -p %{buildroot}%{_rpmluadir}/fedora/srpm +install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm %{SOURCE6} + %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile @@ -71,6 +75,7 @@ install -m 644 %{SOURCE5} \ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmluadir}/fedora/srpm/python.lua %files -n python2-rpm-macros %{rpmmacrodir}/macros.python2 diff --git a/python.lua b/python.lua new file mode 100644 index 0000000..d166171 --- /dev/null +++ b/python.lua @@ -0,0 +1,32 @@ +-- Convenience Lua functions that can be used within Python srpm/rpm macros + +-- Determine alternate names provided from the given name. +-- Used in pythonname provides generator, python_provide and py_provides. +-- There are 2 rules: +-- python3-foo -> python-foo, python3X-foo +-- python3X-foo -> python-foo, python3-foo +-- There is no python-foo -> rule, python-foo packages are version agnostic. +-- Returns a table/array with strings. Empty when no rule matched. +local function python_altnames(name) + local xy = rpm.expand('%{__default_python3_pkgversion}') + local altnames = {} + local replaced + -- NB: dash needs to be escaped! + if name:match('^python3%-') then + for i, prefix in ipairs({'python-', 'python' .. xy .. '-'}) do + replaced = name:gsub('^python3%-', prefix) + table.insert(altnames, replaced) + end + elseif name:match('^python' .. xy .. '%-') then + for i, prefix in ipairs({'python-', 'python3-'}) do + replaced = name:gsub('^python' .. xy .. '%-', prefix) + table.insert(altnames, replaced) + end + end + return altnames +end + + +return { + python_altnames = python_altnames, +} From a5778bf4f24dc02f7ac3b5734de70deb2444fd05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 15:50:54 +0200 Subject: [PATCH 085/218] Reorganize the spec --- python-rpm-macros.spec | 68 ++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4f635f3..94dad05 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -5,19 +5,27 @@ Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 License: MIT and Python -Source0: macros.python -Source1: macros.python-srpm -Source2: macros.python2 -Source3: macros.python3 -Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py -Source6: python.lua + +# Macros: +Source101: macros.python +Source102: macros.python-srpm +Source103: macros.python2 +Source104: macros.python3 +Source105: macros.pybytecompile + +# Lua files +Source201: python.lua + +# Python code +%global compileall2_version 0.7.1 +Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py BuildArch: noarch -# For %%python3_pkgversion used in %%python_provide and compileall2.py -Requires: python-srpm-macros >= 3-46 -Obsoletes: python-macros < 3 -Provides: python-macros = %{version}-%{release} + +# For %%__default_python3_pkgversion used in %%python_provide +# For python.lua +# For compileall2.py +Requires: python-srpm-macros %description This package contains the unversioned Python RPM macros, that most @@ -26,26 +34,40 @@ implementations should rely on. You should not need to install this package manually as the various python?-devel packages require it. So install a python-devel package instead. + %package -n python-srpm-macros Summary: RPM macros for building Python source packages + +# For directory structure and flags macros Requires: redhat-rpm-config +# We bundle our own software here :/ +Provides: bundled(python3dist(compileall2)) = %{compileall2_version} + %description -n python-srpm-macros RPM macros for building Python source packages. + %package -n python2-rpm-macros Summary: RPM macros for building Python 2 packages -Requires: python-srpm-macros >= 3-38 + +# For %%__python2 and %%python2 +Requires: python-srpm-macros + +# For %%py_setup Requires: python-rpm-macros -# Would need to be different for each release - worth it? -#Conflicts: python2-devel < 2.7.11-3 %description -n python2-rpm-macros RPM macros for building Python 2 packages. + %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages -Requires: python-srpm-macros >= 3-38 + +# For %%__python3 and %%python3 +Requires: python-srpm-macros + +# For %%py_setup Requires: python-rpm-macros %description -n python3-rpm-macros @@ -53,20 +75,20 @@ RPM macros for building Python 3 packages. %prep +%autosetup -c -T +cp -a %{sources} . -%build %install mkdir -p %{buildroot}%{rpmmacrodir} -install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ - %{buildroot}%{rpmmacrodir}/ - -mkdir -p %{buildroot}%{_rpmconfigdir}/redhat -install -m 644 %{SOURCE5} \ - %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 macros.* %{buildroot}%{rpmmacrodir}/ mkdir -p %{buildroot}%{_rpmluadir}/fedora/srpm -install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm %{SOURCE6} +install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua + +mkdir -p %{buildroot}%{_rpmconfigdir}/redhat +install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ + %files %{rpmmacrodir}/macros.python From 125134cf84806449e1d1d54286c4f7410823e21f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 16:29:21 +0200 Subject: [PATCH 086/218] Add functions to be used in %py_provides and the provides generator --- python.lua | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python.lua b/python.lua index d166171..45cd366 100644 --- a/python.lua +++ b/python.lua @@ -27,6 +27,36 @@ local function python_altnames(name) end +-- For any given name and epoch-version-release, return provides except self. +-- Uses python_altnames under the hood +-- Returns a table/array with strings. +local function python_altprovides(name, evr) + local altprovides = {} + for i, altname in ipairs(python_altnames(name)) do + table.insert(altprovides, altname .. ' = ' .. evr) + end + return altprovides +end + + +-- Like python_altprovides but only return something once. +-- For each argument can only be used once, returns nil otherwise. +local function python_altprovides_once(name, evr) + -- global cache that tells what provides were already processed + if __python_altnames_provides_beenthere == nil then + __python_altnames_provides_beenthere = {} + end + if __python_altnames_provides_beenthere[name .. ' ' .. evr] == nil then + __python_altnames_provides_beenthere[name .. ' ' .. evr] = true + return python_altprovides(name, evr) + else + return nil + end +end + + return { python_altnames = python_altnames, + python_altprovides = python_altprovides, + python_altprovides_once = python_altprovides_once, } From 8fea79b1ec48499073c55242e7b70a8f86c8132b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 18:48:51 +0200 Subject: [PATCH 087/218] Implement %py_provides --- macros.python-srpm | 21 +++++++++++++++++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 67e53c9..19516ba 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -137,3 +137,24 @@ \ print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext) } + +%py_provides() %{lua: + local python = require 'fedora.srpm.python' + local name = rpm.expand('%1') + if name == '%1' then + rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}') + end + local evr = rpm.expand('%2') + if evr == '%2' then + evr = rpm.expand('%{?epoch:%{epoch}:}%{version}-%{release}') + end + print('Provides: ' .. name .. ' = ' .. evr .. '\\n') + local provides = python.python_altprovides_once(name, evr) + -- provides is either an array/table or nil + -- nil means the function was already called with the same arguments + if provides then + for i, provide in ipairs(provides) do + print('Provides: ' .. provide .. '\\n') + end + end +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 94dad05..024a45d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 4%{?dist} +Release: 5%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue May 05 2020 Miro Hrončok - 3.8-5 +- Implement %%py_provides + * Mon May 04 2020 Tomas Hrnciar - 3.8-4 - Make %%py3_install_wheel macro remove direct_url.json file created by PEP 610. - https://discuss.python.org/t/pep-610-usage-guidelines-for-linux-distributions/4012 From 5fe974759af7c0a62540680b361e5a317cf1502b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 18:54:02 +0200 Subject: [PATCH 088/218] Make %py_provides work repeatedly --- macros.python-srpm | 10 +++------- python.lua | 6 ++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 19516ba..b3a1247 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -149,12 +149,8 @@ evr = rpm.expand('%{?epoch:%{epoch}:}%{version}-%{release}') end print('Provides: ' .. name .. ' = ' .. evr .. '\\n') - local provides = python.python_altprovides_once(name, evr) - -- provides is either an array/table or nil - -- nil means the function was already called with the same arguments - if provides then - for i, provide in ipairs(provides) do - print('Provides: ' .. provide .. '\\n') - end + local provides = python.python_altprovides(name, evr) + for i, provide in ipairs(provides) do + print('Provides: ' .. provide .. '\\n') end } diff --git a/python.lua b/python.lua index 45cd366..8766c91 100644 --- a/python.lua +++ b/python.lua @@ -31,6 +31,11 @@ end -- Uses python_altnames under the hood -- Returns a table/array with strings. local function python_altprovides(name, evr) + -- global cache that tells what provides were already processed + if __python_altnames_provides_beenthere == nil then + __python_altnames_provides_beenthere = {} + end + __python_altnames_provides_beenthere[name .. ' ' .. evr] = true local altprovides = {} for i, altname in ipairs(python_altnames(name)) do table.insert(altprovides, altname .. ' = ' .. evr) @@ -41,6 +46,7 @@ end -- Like python_altprovides but only return something once. -- For each argument can only be used once, returns nil otherwise. +-- Previous usage of python_altprovides counts as well. local function python_altprovides_once(name, evr) -- global cache that tells what provides were already processed if __python_altnames_provides_beenthere == nil then From f5bea8c7b747063345fb07ed1449f3f65b0af422 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Apr 2020 19:28:31 +0200 Subject: [PATCH 089/218] Fedora CI: Add eval tests for %python_provide and %py_provides --- tests/.gitignore | 1 + tests/test_evals.py | 124 ++++++++++++++++++++++++++++++++++++++++++++ tests/tests.yml | 23 ++++++++ 3 files changed, 148 insertions(+) create mode 100644 tests/.gitignore create mode 100644 tests/test_evals.py create mode 100644 tests/tests.yml diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..5732c0f --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +__*__/ diff --git a/tests/test_evals.py b/tests/test_evals.py new file mode 100644 index 0000000..e1d91e7 --- /dev/null +++ b/tests/test_evals.py @@ -0,0 +1,124 @@ +import subprocess +import sys + +XY = f'{sys.version_info[0]}{sys.version_info[1]}' + + +def rpm_eval(expression, **kwargs): + cmd = ['rpmbuild'] + for var, value in kwargs.items(): + cmd += ['--define', f'{var} {value}'] + cmd += ['--eval', expression] + cp = subprocess.run(cmd, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert cp.returncode == 0, cp.stderr + return cp.stdout.strip().splitlines() + + +def test_python_provide_python(): + assert rpm_eval('%python_provide python-foo') == [] + + +def test_python_provide_python3(): + lines = rpm_eval('%python_provide python3-foo', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert len(lines) == 3 + + +def test_python_provide_python3_epoched(): + lines = rpm_eval('%python_provide python3-foo', epoch='1', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + assert len(lines) == 3 + + +def test_python_provide_python3X(): + lines = rpm_eval(f'%python_provide python{XY}-foo', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert len(lines) == 3 + + +def test_python_provide_python3X_epoched(): + lines = rpm_eval(f'%python_provide python{XY}-foo', epoch='1', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert 'Provides: python3-foo = 1:6-1.fc66' in lines + assert len(lines) == 3 + + +def test_python_provide_doubleuse(): + lines = rpm_eval('%{python_provide python3-foo}%{python_provide python3-foo}', + version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert len(lines) == 6 + assert len(set(lines)) == 3 + + +def test_py_provides_python(): + lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66') + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 1 + + +def test_py_provides_whatever(): + lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66') + assert 'Provides: whatever = 6-1.fc66' in lines + assert len(lines) == 1 + + +def test_py_provides_python3(): + lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66') + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert len(lines) == 3 + + +def test_py_provides_python3_epoched(): + lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66') + assert 'Provides: python3-foo = 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + assert len(lines) == 3 + + +def test_py_provides_python3X(): + lines = rpm_eval(f'%py_provides python{XY}-foo', version='6', release='1.fc66') + assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert len(lines) == 3 + + +def test_py_provides_python3X_epoched(): + lines = rpm_eval(f'%py_provides python{XY}-foo', epoch='1', version='6', release='1.fc66') + assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert 'Provides: python3-foo = 1:6-1.fc66' in lines + assert len(lines) == 3 + + +def test_py_provides_doubleuse(): + lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}', + version='6', release='1.fc66') + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert len(lines) == 6 + assert len(set(lines)) == 3 + + +def test_py_provides_with_evr(): + lines = rpm_eval('%py_provides python3-foo 123', + version='6', release='1.fc66') + assert 'Provides: python3-foo = 123' in lines + assert 'Provides: python-foo = 123' in lines + assert f'Provides: python{XY}-foo = 123' in lines + assert len(lines) == 3 diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..10c35b7 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,23 @@ +--- +- hosts: localhost + tags: + - classic + tasks: + - dnf: + name: "*" + state: latest + +- hosts: localhost + roles: + - role: standard-test-basic + tags: + - classic + tests: + - pytest: + dir: . + run: pytest -v + required_packages: + - rpm-build + - python-rpm-macros + - python3-pytest + From 25355e4a16b62f22a047828bcb81752dcbae94a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 5 May 2020 01:38:35 +0200 Subject: [PATCH 090/218] Reuse python.python_altprovides in %python_provide This way, the generator won't re-add the same provide again, keeping rpmlint happy even with the old macro. --- macros.python | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/macros.python b/macros.python index 0044fae..15abf48 100644 --- a/macros.python +++ b/macros.python @@ -59,10 +59,10 @@ end local package = rpm.expand("%{?1}") local vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") - local altnames = python.python_altnames(package) + local provides = python.python_altprovides(package, vr) if (string.starts(package, "python3-")) then - for i, altname in ipairs(altnames) do - print("\\nProvides: " .. altname .. " = " .. vr) + for i, provide in ipairs(provides) do + print("\\nProvides: " .. provide) end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then @@ -71,8 +71,8 @@ print(" < " .. vr) end elseif (string.starts(package, "python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-")) then - for i, altname in ipairs(altnames) do - print("\\nProvides: " .. altname .. " = " .. vr) + for i, provide in ipairs(provides) do + print("\\nProvides: " .. provide) end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then From 76aecd9ad76021a415192e0b17ecda20771e0656 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 6 May 2020 01:31:46 +0200 Subject: [PATCH 091/218] Require recent enough SRPM macros from RPM macros, to prevent missing Lua files %pyhon_provide in python-rpm-macros uses new Lua functions from python-srpm-macros People have python-srpm-macros pre-installed, and it is not autoupdated, hence they can have older versions installed and it blows up. --- python-rpm-macros.spec | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 024a45d..bc687a7 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 5%{?dist} +Release: 6%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -25,7 +25,7 @@ BuildArch: noarch # For %%__default_python3_pkgversion used in %%python_provide # For python.lua # For compileall2.py -Requires: python-srpm-macros +Requires: python-srpm-macros >= 3.8-5 %description This package contains the unversioned Python RPM macros, that most @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue May 05 2020 Miro Hrončok - 3.8-6 +- Require recent enough SRPM macros from RPM macros, to prevent missing Lua files + * Tue May 05 2020 Miro Hrončok - 3.8-5 - Implement %%py_provides From 5f3e4d630051d2b8c47431dafdd8752f9882af46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 7 May 2020 17:28:36 +0200 Subject: [PATCH 092/218] Change %__default_python3_pkgversion from 38 to 3.8 See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/message/VIUS7WMQMDX6H2WEIH7TVTMBB6SUHY7E/ --- macros.python | 2 +- macros.python-srpm | 27 ++++++++++++++++++++++++--- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 28 ++++++++++++++-------------- 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/macros.python b/macros.python index 15abf48..4fdab09 100644 --- a/macros.python +++ b/macros.python @@ -77,7 +77,7 @@ --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") - print(string.sub(package,10,string.len(package))) + print(string.sub(package,11,string.len(package))) print(" < " .. vr) end elseif (string.starts(package, "python")) then diff --git a/macros.python-srpm b/macros.python-srpm index b3a1247..ed41df4 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -25,10 +25,31 @@ end\ } -# This specifies what packages are equal to the python3-... packages, in python_provide macro -# E.g. in Fedora 32, python38-foo will provide python3-foo and vice versa +# There are multiple Python 3 versions packaged, but only one can be the "main" version +# That means that it owns the "python3" namespace: +# - python3 package name +# - /usr/bin/python3 command +# - python3-foo packages are meant for this version +# Other versions of Python 3 always contain the version in the namespace: +# - python3.XX package name +# - /usr/bin/python3.XX command +# - python3.XX-foo packages (if allowed) +# +# Python spec files use the version defined here to determine defaults for the +# %%py_provides and %%python_provide macros, as well as for the "pythonname" generator that +# provides python3-foo for python3.XX-foo and vice versa for the default "main" version. +# E.g. in Fedora 32, python3.8-foo will provide python3-foo, +# python3-foo will provide python3.8-foo. +# +# There are two macros: +# +# This always contains the major.minor version (with dots), default for %%python3_version. %__default_python3_version 3.8 -%__default_python3_pkgversion %(echo %__default_python3_version | sed 's/\\.//') +# +# The pkgname version that determines the alternative provide name (e.g. python3.8-foo), +# set to the same as above, but historically hasn't included the dot. +# This is left intentionally a separate macro, in case the naming convention ever changes. +%__default_python3_pkgversion %__default_python3_version # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bc687a7..f10aafb 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 6%{?dist} +Release: 7%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu May 07 2020 Miro Hrončok - 3.8-7 +- Change %%__default_python3_pkgversion from 38 to 3.8 + * Tue May 05 2020 Miro Hrončok - 3.8-6 - Require recent enough SRPM macros from RPM macros, to prevent missing Lua files diff --git a/tests/test_evals.py b/tests/test_evals.py index e1d91e7..055426b 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,7 +1,7 @@ import subprocess import sys -XY = f'{sys.version_info[0]}{sys.version_info[1]}' +X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' def rpm_eval(expression, **kwargs): @@ -23,7 +23,7 @@ def test_python_provide_python3(): lines = rpm_eval('%python_provide python3-foo', version='6', release='1.fc66') assert 'Obsoletes: python-foo < 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert len(lines) == 3 @@ -31,12 +31,12 @@ def test_python_provide_python3_epoched(): lines = rpm_eval('%python_provide python3-foo', epoch='1', version='6', release='1.fc66') assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines assert len(lines) == 3 def test_python_provide_python3X(): - lines = rpm_eval(f'%python_provide python{XY}-foo', version='6', release='1.fc66') + lines = rpm_eval(f'%python_provide python{X_Y}-foo', version='6', release='1.fc66') assert 'Obsoletes: python-foo < 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert 'Provides: python3-foo = 6-1.fc66' in lines @@ -44,7 +44,7 @@ def test_python_provide_python3X(): def test_python_provide_python3X_epoched(): - lines = rpm_eval(f'%python_provide python{XY}-foo', epoch='1', version='6', release='1.fc66') + lines = rpm_eval(f'%python_provide python{X_Y}-foo', epoch='1', version='6', release='1.fc66') assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert 'Provides: python3-foo = 1:6-1.fc66' in lines @@ -56,7 +56,7 @@ def test_python_provide_doubleuse(): version='6', release='1.fc66') assert 'Obsoletes: python-foo < 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert len(lines) == 6 assert len(set(lines)) == 3 @@ -77,7 +77,7 @@ def test_py_provides_python3(): lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66') assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert len(lines) == 3 @@ -85,21 +85,21 @@ def test_py_provides_python3_epoched(): lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66') assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines assert len(lines) == 3 def test_py_provides_python3X(): - lines = rpm_eval(f'%py_provides python{XY}-foo', version='6', release='1.fc66') - assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66') + assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert 'Provides: python3-foo = 6-1.fc66' in lines assert len(lines) == 3 def test_py_provides_python3X_epoched(): - lines = rpm_eval(f'%py_provides python{XY}-foo', epoch='1', version='6', release='1.fc66') - assert f'Provides: python{XY}-foo = 1:6-1.fc66' in lines + lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66') + assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert len(lines) == 3 @@ -110,7 +110,7 @@ def test_py_provides_doubleuse(): version='6', release='1.fc66') assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines - assert f'Provides: python{XY}-foo = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert len(lines) == 6 assert len(set(lines)) == 3 @@ -120,5 +120,5 @@ def test_py_provides_with_evr(): version='6', release='1.fc66') assert 'Provides: python3-foo = 123' in lines assert 'Provides: python-foo = 123' in lines - assert f'Provides: python{XY}-foo = 123' in lines + assert f'Provides: python{X_Y}-foo = 123' in lines assert len(lines) == 3 From 0d3f1e6b7457f57fa1e2d4c9976665574345fb5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 6 May 2020 13:21:32 +0200 Subject: [PATCH 093/218] Implement %pytest See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/XLPDSH362PJKMJCAYOXNJNV53Y66EF6B/ --- macros.python3 | 9 +++++++++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index 7af56f1..a093d0f 100644 --- a/macros.python3 +++ b/macros.python3 @@ -62,3 +62,12 @@ print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") end } + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest +%pytest %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %__pytest} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index f10aafb..e4c9c67 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.8 -Release: 7%{?dist} +Release: 8%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon May 11 2020 Miro Hrončok - 3.8-8 +- Implement %%pytest + * Thu May 07 2020 Miro Hrončok - 3.8-7 - Change %%__default_python3_pkgversion from 38 to 3.8 diff --git a/tests/test_evals.py b/tests/test_evals.py index 055426b..89848ca 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -122,3 +122,13 @@ def test_py_provides_with_evr(): assert 'Provides: python-foo = 123' in lines assert f'Provides: python{X_Y}-foo = 123' in lines assert len(lines) == 3 + + +def test_pytest_passes_options_naturally(): + lines = rpm_eval('%pytest -k foo') + assert '/usr/bin/pytest -k foo' in lines[-1] + + +def test_pytest_different_command(): + lines = rpm_eval('%pytest', __pytest='pytest-3') + assert 'pytest-3' in lines[-1] From 4569c61d8d655e2c435308800632101f8eae7cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 11 May 2020 19:02:37 +0200 Subject: [PATCH 094/218] Strip tildes from %version in %pypi_source by default, add tests --- macros.python-srpm | 4 ++-- python-rpm-macros.spec | 1 + tests/test_evals.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index ed41df4..45f4fca 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -119,7 +119,7 @@ # Accepts zero to three arguments: # 1: The PyPI project name, defaulting to %srcname if it is defined, then # %pypi_name if it is defined, then just %name. -# 2: The PYPI version, defaulting to %version. +# 2: The PYPI version, defaulting to %version with tildes stripped. # 3: The file extension, defaulting to "tar.gz". (A period will be added # automatically.) # Requires %__pypi_url and %__pypi_default_extension to be defined. @@ -146,7 +146,7 @@ \ -- If no second argument, use %version if ver == '%2' then - ver = rpm.expand('%version') + ver = rpm.expand('%version'):gsub('~', '') end \ -- If no third argument, use the preset default extension diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e4c9c67..1434907 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -109,6 +109,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog * Mon May 11 2020 Miro Hrončok - 3.8-8 - Implement %%pytest +- Strip tildes from %%version in %%pypi_source by default * Thu May 07 2020 Miro Hrončok - 3.8-7 - Change %%__default_python3_pkgversion from 38 to 3.8 diff --git a/tests/test_evals.py b/tests/test_evals.py index 89848ca..d4d2284 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -132,3 +132,57 @@ def test_pytest_passes_options_naturally(): def test_pytest_different_command(): lines = rpm_eval('%pytest', __pytest='pytest-3') assert 'pytest-3' in lines[-1] + + +def test_pypi_source_default_name(): + url = rpm_eval('%pypi_source', + name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_srcname(): + url = rpm_eval('%pypi_source', + name='python-foo', srcname='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_pypi_name(): + url = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_name_uppercase(): + url = rpm_eval('%pypi_source', + name='Foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz' + + +def test_pypi_source_provided_name(): + url = rpm_eval('%pypi_source foo', + name='python-bar', pypi_name='bar', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_provided_name_version(): + url = rpm_eval('%pypi_source foo 6', + name='python-bar', pypi_name='bar', version='3')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_provided_name_version_ext(): + url = rpm_eval('%pypi_source foo 6 zip', + name='python-bar', pypi_name='bar', version='3')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip' + + +def test_pypi_source_prerelease(): + url = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6~b2')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz' + + +def test_pypi_source_explicit_tilde(): + url = rpm_eval('%pypi_source foo 6~6', + name='python-foo', pypi_name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz' From 72371929c5751041c25e5bb57bdddbc6716fb54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 18 May 2020 18:58:39 +0200 Subject: [PATCH 095/218] Implement %pyX_shebang_fix See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/UGCMDDG3S32U7JJK36OEZNHLUVQQRG3M/ --- macros.python | 2 ++ macros.python2 | 2 ++ macros.python3 | 2 ++ python-rpm-macros.spec | 1 + tests/test_evals.py | 25 ++++++++++++++++++++++++- tests/tests.yml | 1 + 6 files changed, 32 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 4fdab09..713f746 100644 --- a/macros.python +++ b/macros.python @@ -8,6 +8,8 @@ %py_setup setup.py %py_shbang_opts -s %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) +%py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) +%py_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python} -k%{?py_shebang_flags:a %py_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 82d7428..0e4b0cb 100644 --- a/macros.python2 +++ b/macros.python2 @@ -5,6 +5,8 @@ %py2_shbang_opts -s %py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) +%py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-}) +%py2_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python2} -k%{?py2_shebang_flags:a %py2_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index a093d0f..681f984 100644 --- a/macros.python3 +++ b/macros.python3 @@ -7,6 +7,8 @@ %py3_shbang_opts -s %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) +%py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) +%py3_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python3} -k%{?py3_shebang_flags:a %py3_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 1434907..fec0a6c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -109,6 +109,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog * Mon May 11 2020 Miro Hrončok - 3.8-8 - Implement %%pytest +- Implement %%pyX_shebang_fix - Strip tildes from %%version in %%pypi_source by default * Thu May 07 2020 Miro Hrončok - 3.8-7 diff --git a/tests/test_evals.py b/tests/test_evals.py index d4d2284..3ff8144 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -7,7 +7,10 @@ X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' def rpm_eval(expression, **kwargs): cmd = ['rpmbuild'] for var, value in kwargs.items(): - cmd += ['--define', f'{var} {value}'] + if value is None: + cmd += ['--undefine', var] + else: + cmd += ['--define', f'{var} {value}'] cmd += ['--eval', expression] cp = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -186,3 +189,23 @@ def test_pypi_source_explicit_tilde(): url = rpm_eval('%pypi_source foo 6~6', name='python-foo', pypi_name='foo', version='6')[0] assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz' + + +def test_py3_shebang_fix(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka s arg1 arg2 arg3' + + +def test_py3_shebang_fix_custom_flags(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags='Es')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka Es arg1 arg2 arg3' + + +def test_py3_shebang_fix_empty_flags(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags=None)[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -k arg1 arg2 arg3' + + +def test_py_shebang_fix_custom(): + cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy -ka s arg1 arg2 arg3' diff --git a/tests/tests.yml b/tests/tests.yml index 10c35b7..9fb4da3 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -19,5 +19,6 @@ required_packages: - rpm-build - python-rpm-macros + - python3-rpm-macros - python3-pytest From 5919708f6c5c2e9528e9f8f6ac8efd0f7a85b224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 21 May 2020 17:39:01 +0200 Subject: [PATCH 096/218] https://fedoraproject.org/wiki/Changes/Python3.9 Also switch to PEP 503 based %py_dist_name (see rhbz#1791530) --- macros.python-srpm | 10 +++++----- python-rpm-macros.spec | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 45f4fca..15a7193 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -38,15 +38,15 @@ # Python spec files use the version defined here to determine defaults for the # %%py_provides and %%python_provide macros, as well as for the "pythonname" generator that # provides python3-foo for python3.XX-foo and vice versa for the default "main" version. -# E.g. in Fedora 32, python3.8-foo will provide python3-foo, -# python3-foo will provide python3.8-foo. +# E.g. in Fedora 33, python3.9-foo will provide python3-foo, +# python3-foo will provide python3.9-foo. # # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.8 +%__default_python3_version 3.9 # -# The pkgname version that determines the alternative provide name (e.g. python3.8-foo), +# The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. # This is left intentionally a separate macro, in case the naming convention ever changes. %__default_python3_pkgversion %__default_python3_version @@ -70,7 +70,7 @@ # Converts Python dist name to a canonical format %py_dist_name() %{lua:\ name = rpm.expand("%{?1:%{1}}");\ - canonical = string.gsub(string.lower(name), "[^%w%.]+", "-");\ + canonical = string.gsub(string.lower(name), "%W+", "-");\ print(canonical);\ } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index fec0a6c..95cd352 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros -Version: 3.8 -Release: 8%{?dist} +Version: 3.9 +Release: 1%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu May 21 2020 Miro Hrončok - 3.9-1 +- https://fedoraproject.org/wiki/Changes/Python3.9 +- Switch the %%py_dist_name macro to convert dots (".") into dashes as defined in PEP 503 (#1791530) + * Mon May 11 2020 Miro Hrončok - 3.8-8 - Implement %%pytest - Implement %%pyX_shebang_fix From bae52eafbe3b20d1d33158f8b66301e62dc03170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sun, 31 May 2020 00:28:19 +0200 Subject: [PATCH 097/218] Require the exact same version-release of other subpackages of this package To avoid mock using a local copy of python-srpm-macros from the buildroot cache and having Python version defined to 3.8 until the cache expires. Also, other surprises happened in the past, so we stop playing it cool and actually follow the guidelines here. --- python-rpm-macros.spec | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 95cd352..5e765c8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 1%{?dist} +Release: 2%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -25,7 +25,7 @@ BuildArch: noarch # For %%__default_python3_pkgversion used in %%python_provide # For python.lua # For compileall2.py -Requires: python-srpm-macros >= 3.8-5 +Requires: python-srpm-macros = %{version}-%{release} %description This package contains the unversioned Python RPM macros, that most @@ -52,10 +52,10 @@ RPM macros for building Python source packages. Summary: RPM macros for building Python 2 packages # For %%__python2 and %%python2 -Requires: python-srpm-macros +Requires: python-srpm-macros = %{version}-%{release} # For %%py_setup -Requires: python-rpm-macros +Requires: python-rpm-macros = %{version}-%{release} %description -n python2-rpm-macros RPM macros for building Python 2 packages. @@ -65,10 +65,10 @@ RPM macros for building Python 2 packages. Summary: RPM macros for building Python 3 packages # For %%__python3 and %%python3 -Requires: python-srpm-macros +Requires: python-srpm-macros = %{version}-%{release} # For %%py_setup -Requires: python-rpm-macros +Requires: python-rpm-macros = %{version}-%{release} %description -n python3-rpm-macros RPM macros for building Python 3 packages. @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Sat May 30 2020 Miro Hrončok - 3.9-2 +- Require the exact same version-release of other subpackages of this package + * Thu May 21 2020 Miro Hrončok - 3.9-1 - https://fedoraproject.org/wiki/Changes/Python3.9 - Switch the %%py_dist_name macro to convert dots (".") into dashes as defined in PEP 503 (#1791530) From 985a80572f1f140ee76027d17227105fea0c964d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 27 May 2020 10:30:27 +0200 Subject: [PATCH 098/218] Allow to combine %pycached with other macros (e.g. %exclude or %ghost) Previous implementation allowed for only one argument to be passed to the %pycached macro, which made it impossible to combine it with other macros. Current implementation allows to pass other macros as arguments to %pycached. Example: %pycached %exclude /path/to/foo.py For macro expansion limitations, the opposite order is not possible. That is to be documented in the guidelines: https://pagure.io/packaging-committee/pull-request/986 Added some tests. Resolves https://bugzilla.redhat.com/show_bug.cgi?id=1838992 Co-authored-by: Marcel Plch --- macros.python3 | 2 +- python-rpm-macros.spec | 5 +++- tests/test_evals.py | 59 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 6 deletions(-) diff --git a/macros.python3 b/macros.python3 index 681f984..765e56b 100644 --- a/macros.python3 +++ b/macros.python3 @@ -53,7 +53,7 @@ # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. %pycached() %{lua: - path = rpm.expand("%{?1}") + path = rpm.expand("%{?*}") if (string.sub(path, "-3") ~= ".py") then rpm.expand("%{error:%%pycached can only be used with paths explicitly ending with .py}") else diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5e765c8..6f05af8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 2%{?dist} +Release: 3%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu Jun 11 2020 Miro Hrončok - 3.9-3 +- Allow to combine %%pycached with other macros (e.g. %%exclude or %%ghost) (#1838992) + * Sat May 30 2020 Miro Hrončok - 3.9-2 - Require the exact same version-release of other subpackages of this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 3ff8144..fbe76b5 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,10 +1,12 @@ +import os import subprocess import sys X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' +XY = f'{sys.version_info[0]}{sys.version_info[1]}' -def rpm_eval(expression, **kwargs): +def rpm_eval(expression, fails=False, **kwargs): cmd = ['rpmbuild'] for var, value in kwargs.items(): if value is None: @@ -12,9 +14,12 @@ def rpm_eval(expression, **kwargs): else: cmd += ['--define', f'{var} {value}'] cmd += ['--eval', expression] - cp = subprocess.run(cmd, text=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert cp.returncode == 0, cp.stderr + cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if fails: + assert cp.returncode != 0, cp.stdout + elif fails is not None: + assert cp.returncode == 0, cp.stdout return cp.stdout.strip().splitlines() @@ -209,3 +214,49 @@ def test_py3_shebang_fix_empty_flags(): def test_py_shebang_fix_custom(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[0] assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy -ka s arg1 arg2 arg3' + + +def test_pycached_in_sitelib(): + lines = rpm_eval('%pycached %{python3_sitelib}/foo*.py') + assert lines == [ + f'/usr/lib/python{X_Y}/site-packages/foo*.py', + f'/usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_in_sitearch(): + lines = rpm_eval('%pycached %{python3_sitearch}/foo*.py') + lib = rpm_eval('%_lib')[0] + assert lines == [ + f'/usr/{lib}/python{X_Y}/site-packages/foo*.py', + f'/usr/{lib}/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_in_36(): + lines = rpm_eval('%pycached /usr/lib/python3.6/site-packages/foo*.py') + assert lines == [ + '/usr/lib/python3.6/site-packages/foo*.py', + '/usr/lib/python3.6/site-packages/__pycache__/foo*.cpython-36{,.opt-?}.pyc' + ] + + +def test_pycached_in_custom_dir(): + lines = rpm_eval('%pycached /bar/foo*.py') + assert lines == [ + '/bar/foo*.py', + '/bar/__pycache__/foo*.cpython-3*{,.opt-?}.pyc' + ] + + +def test_pycached_with_exclude(): + lines = rpm_eval('%pycached %exclude %{python3_sitelib}/foo*.py') + assert lines == [ + f'%exclude /usr/lib/python{X_Y}/site-packages/foo*.py', + f'%exclude /usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_fails_with_extension_glob(): + lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True) + assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py' From f77cb3e9dd9b452b19582a5765178e3a3ae8c084 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 20 May 2020 15:06:16 +0200 Subject: [PATCH 099/218] No more automagic Python bytecompilation (phase 3) --- brp-python-bytecompile | 41 +++++++++-------------------------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 4727feb..f8ca447 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -1,6 +1,15 @@ #!/bin/bash errors_terminate=$2 + +# Usage of %_python_bytecompile_extra is not allowed anymore +# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 +# Therefore $1 ($default_python) is not needed and is invoked with "" by default. +# $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 +if [ 0$extra -eq 1 ]; then + echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr + exit 1 +fi # If using normal root, avoid changing anything. if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then @@ -110,35 +119,3 @@ do exit 1 fi done - - -# Handle other locations in the filesystem using the default python implementation -# if extra is set to 0, don't do this -if [ 0$extra -eq 0 ]; then - exit 0 -fi - -# If we don't have a default python interpreter, we cannot proceed -default_python=${1:-/usr/bin/python} -if [ ! -x "$default_python" ]; then - exit 0 -fi - -# Figure out if there are files to be bytecompiled with the default_python at all -# this prevents unnecessary default_python invocation -find "$RPM_BUILD_ROOT" -type f -name "*.py" | grep -Ev "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" || exit 0 - -# Generate normal (.pyc) byte-compiled files. -python_bytecompile "" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" -if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then - # One or more of the files had a syntax error - exit 1 -fi - -# Generate optimized (.pyo) byte-compiled files. -python_bytecompile "-O" $default_python "/bin/|/sbin/|/usr/lib(64)?/python[0-9]\.[0-9]|/usr/share/doc" "$RPM_BUILD_ROOT" "$depth" "/" -if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then - # One or more of the files had a syntax error - exit 1 -fi -exit 0 From 437166cca727048be145b329ba53383f20b816c8 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Wed, 20 May 2020 15:06:26 +0200 Subject: [PATCH 100/218] Remove trailing whitespace --- brp-python-bytecompile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index f8ca447..eabc39a 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -94,7 +94,7 @@ fi # # Support this by assuming that below each /usr/lib/python$VERSION/, all # .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. -# +# # For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 # and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 From 8e9c3d8bbecf840e7463c8790c085b7c90fc881e Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 5 Jun 2020 08:59:16 +0200 Subject: [PATCH 101/218] Use compileall from stdlib for Python >= 3.9 All enhancements from compileall2 are merged in Python 3.9. --- brp-python-bytecompile | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index eabc39a..c6afd1e 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -24,14 +24,15 @@ if [ -z "$depth" -o "$depth" -le "1" ]; then exit 0 fi -# This function now implements Python byte-compilation in two different ways: -# Python >= 3.4 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# This function now implements Python byte-compilation in three different ways: +# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks # When we drop support for Python 2, we'd be able to use all compileall2 features like: # - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir # - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options # - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() # These changes will make this script much simpler +# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. function python_bytecompile() { local options=$1 @@ -43,10 +44,25 @@ function python_bytecompile() python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + # + # Python 3.9 and higher + # + if [ "$python_version" -ge 39 ]; then + + [ ! -z $exclude ] && exclude="-x '$exclude'" + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + $python_binary -B $options -m compileall -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir + # # Python 3.4 and higher # - if [ "$python_version" -ge 34 ]; then + elif [ "$python_version" -ge 34 ]; then [ ! -z $exclude ] && exclude="-x '$exclude'" # /usr/lib/rpm/redhat/ contains compileall2 Python module From 3a211cc91bf625edc868d839c7b1502f024befb5 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 5 Jun 2020 08:52:58 +0200 Subject: [PATCH 102/218] Use compileall from stdlib for Python >= 3.9 --- macros.pybytecompile | 17 +++++++++++++++-- python-rpm-macros.spec | 5 ++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index 38ac84a..f34716b 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -28,9 +28,22 @@ py3_byte_compile () {\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ }\ \ +py39_byte_compile () {\ + python_binary="%1"\ + bytecode_compilation_path="%2"\ + $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ +}\ +\ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ -# compileall2 Python module is not compatible with Python < 3.4 \ -if [ "$python_version" -ge 34 ]; then py3_byte_compile "%1" "%2"; else py2_byte_compile "%1" "%2"; fi +# compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ +# and it was merged back to stdlib in Python >= 3.9 \ +if [ "$python_version" -ge 39 ]; then \ +py39_byte_compile "%1" "%2"; \ +elif [ "$python_version" -ge 34 ]; then \ +py3_byte_compile "%1" "%2"; \ +else \ +py2_byte_compile "%1" "%2"; \ +fi diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 6f05af8..523d314 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 3%{?dist} +Release: 4%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Jun 16 2020 Lumír Balhar - 3.9-4 +- Use compileall from stdlib for Python >= 3.9 + * Thu Jun 11 2020 Miro Hrončok - 3.9-3 - Allow to combine %%pycached with other macros (e.g. %%exclude or %%ghost) (#1838992) From 763d24cc5cadd38959177e0c68c9315fd33f19e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 9 Jul 2020 00:49:15 +0200 Subject: [PATCH 103/218] Add %python_extras_subpkg See https://fedoraproject.org/wiki/Changes/PythonExtras --- macros.python-srpm | 48 ++++++++++++++++++++++++++ python-rpm-macros.spec | 6 +++- tests/test_evals.py | 76 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 15a7193..ff1c372 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -175,3 +175,51 @@ print('Provides: ' .. provide .. '\\n') end } + +%python_extras_subpkg(n:i:f:F) %{expand:%{lua: + local option_n = '-n (name of the base package)' + local option_i = '-i (buildroot path to metadata)' + local option_f = '-f (builddir path to a filelist)' + local option_F = '-F (skip %%files section)' + local value_n = rpm.expand('%{-n*}') + local value_i = rpm.expand('%{-i*}') + local value_f = rpm.expand('%{-f*}') + local value_F = rpm.expand('%{-F}') + local args = rpm.expand('%{*}') + if value_n == '' then + rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') + end + if value_i == '' and value_f == '' and value_F == '' then + rpm.expand('%{error:%%%0: missing option ' .. option_i .. ' or ' .. option_f .. ' or ' .. option_F .. '}') + end + if value_i ~= '' and value_f ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_i .. ' and ' .. option_f .. ' options are not possible}') + end + if value_i ~= '' and value_F ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_i .. ' and ' .. option_F .. ' options are not possible}') + end + if value_f ~= '' and value_F ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') + end + if args == '' then + rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') + end + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + for extras in args:gmatch('%w+') do + local rpmname = value_n .. '+' .. extras + local pkgdef = '%package -n ' .. rpmname + local summary = 'Summary: Metapackage for ' .. value_n .. ': ' .. extras .. ' extras' + local description = '%description -n ' .. rpmname .. '\\\n' .. + 'This is a metapackage bringing in ' .. extras .. ' extras requires for ' .. value_n .. '.\\\n' .. + 'It contains no code, just makes sure the dependencies are installed.\\\n' + local files = '' + if value_i ~= '' then + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + elseif value_f ~= '' then + files = '%files -n ' .. rpmname .. ' -f ' .. value_f + end + for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do + print(line .. '\\\n') + end + end +}} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 523d314..5b6a64c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 4%{?dist} +Release: 5%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Jul 08 2020 Miro Hrončok - 3.9-5 +- Introduce %%python_extras_subpkg +- https://fedoraproject.org/wiki/Changes/PythonExtras + * Tue Jun 16 2020 Lumír Balhar - 3.9-4 - Use compileall from stdlib for Python >= 3.9 diff --git a/tests/test_evals.py b/tests/test_evals.py index fbe76b5..70c7f9f 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,6 +1,7 @@ import os import subprocess import sys +import textwrap X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' XY = f'{sys.version_info[0]}{sys.version_info[1]}' @@ -260,3 +261,78 @@ def test_pycached_with_exclude(): def test_pycached_fails_with_extension_glob(): lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True) assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py' + + +def test_python_extras_subpkg_i(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -i %{python3_sitelib}/*.egg-info toml yaml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + + %files -n python3-setuptools_scm+toml + %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + + %package -n python3-setuptools_scm+yaml + Summary: Metapackage for python3-setuptools_scm: yaml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+yaml + This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + + %files -n python3-setuptools_scm+yaml + %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_f(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -f ghost_filelist toml yaml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + + %files -n python3-setuptools_scm+toml -f ghost_filelist + + %package -n python3-setuptools_scm+yaml + Summary: Metapackage for python3-setuptools_scm: yaml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+yaml + This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + + %files -n python3-setuptools_scm+yaml -f ghost_filelist + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_F(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -F toml yaml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + + + + %package -n python3-setuptools_scm+yaml + Summary: Metapackage for python3-setuptools_scm: yaml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+yaml + This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + It contains no code, just makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected From 59abe832d4d489da3b9f09f280cda098aee39d1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 10 Jul 2020 15:41:39 +0200 Subject: [PATCH 104/218] Adapt %%py_dist_name to keep square brackets So %{py3_dist foo[bar]} works as expected. Add tests. --- macros.python-srpm | 2 +- python-rpm-macros.spec | 1 + tests/test_evals.py | 26 ++++++++++++++++++++++++++ 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index ff1c372..c0d31b0 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -70,7 +70,7 @@ # Converts Python dist name to a canonical format %py_dist_name() %{lua:\ name = rpm.expand("%{?1:%{1}}");\ - canonical = string.gsub(string.lower(name), "%W+", "-");\ + canonical = string.gsub(string.lower(name), "[^%w%[%]]+", "-");\ print(canonical);\ } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5b6a64c..eb381a8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -109,6 +109,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog * Wed Jul 08 2020 Miro Hrončok - 3.9-5 - Introduce %%python_extras_subpkg +- Adapt %%py_dist_name to keep square brackets - https://fedoraproject.org/wiki/Changes/PythonExtras * Tue Jun 16 2020 Lumír Balhar - 3.9-4 diff --git a/tests/test_evals.py b/tests/test_evals.py index 70c7f9f..ce38793 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -3,6 +3,8 @@ import subprocess import sys import textwrap +import pytest + X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' XY = f'{sys.version_info[0]}{sys.version_info[1]}' @@ -24,6 +26,30 @@ def rpm_eval(expression, fails=False, **kwargs): return cp.stdout.strip().splitlines() +@pytest.mark.parametrize('argument, result', [ + ('a', 'a'), + ('a-a', 'a-a'), + ('a_a', 'a-a'), + ('a.a', 'a-a'), + ('a---a', 'a-a'), + ('a-_-a', 'a-a'), + ('a-_-a', 'a-a'), + ('a[b]', 'a[b]'), + ('Aha[Boom]', 'aha[boom]'), + ('a.a[b.b]', 'a-a[b-b]'), +]) +def test_pydist_name(argument, result): + assert rpm_eval(f'%py_dist_name {argument}') == [result] + + +def test_py2_dist(): + assert rpm_eval(f'%py2_dist Aha[Boom] a') == ['python2dist(aha[boom]) python2dist(a)'] + + +def test_py3_dist(): + assert rpm_eval(f'%py3_dist Aha[Boom] a') == ['python3dist(aha[boom]) python3dist(a)'] + + def test_python_provide_python(): assert rpm_eval('%python_provide python-foo') == [] From 69b1b30d532a8c4320956b60aeef1af22845b11c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Jul 2020 18:21:15 +0200 Subject: [PATCH 105/218] Make the unversioned %__python macro error See https://fedoraproject.org/wiki/Changes/PythonMacroError While doing it, make %python macros more consistent with %python3 macros, mostly wrt whitespace but also to use python -m pip over plain pip etc. One significant change is the removal of sleeps from python macros, this could affect packages that use python macros to build for Python 2 while also using python3 macros to build for Python 3. In reality, I consider that unlikely. The sleep in python2 macros stays. The --strip-file-prefix option was already removed from %pyX_install_wheel but we forgot to remove it from %py_install_wheel. --- macros.python | 7 ++----- macros.python-srpm | 14 +++----------- macros.python2 | 4 ++-- macros.python3 | 6 +++--- python-rpm-macros.spec | 7 ++++++- tests/test_evals.py | 30 ++++++++++++++++++++++++++++++ 6 files changed, 46 insertions(+), 22 deletions(-) diff --git a/macros.python b/macros.python index 713f746..2ecb49d 100644 --- a/macros.python +++ b/macros.python @@ -16,19 +16,16 @@ %py_build() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} - sleep 1 } %py_build_egg() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} - sleep 1 } %py_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} - sleep 1 } %py_install() %{expand:\\\ @@ -39,12 +36,12 @@ %py_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python_sitelib} - easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} + %{__python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py_install_wheel() %{expand:\\\ - pip install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps + %{__python} -m pip install -I dist/%{1} --root %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python_sitelib}/*.dist-info %{buildroot}%{python_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/macros.python-srpm b/macros.python-srpm index c0d31b0..fa9f06a 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -11,19 +11,11 @@ %python2 %__python2 %python3 %__python3 -# For backwards compatibility only -# See the comments in https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/22 -%__python /usr/bin/python +# See https://fedoraproject.org/wiki/Changes/PythonMacroError +%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} # Users can use %%python only if they redefined %%__python (e.g. to %%__python3) -%python() %{lua:\ - __python = rpm.expand("%__python")\ - if __python == "/usr/bin/python" then\ - rpm.expand("%{error:Cannot use %%python if %%__python wasn't redefined to something other than /usr/bin/python.}")\ - else\ - print(__python)\ - end\ -} +%python %__python # There are multiple Python 3 versions packaged, but only one can be the "main" version # That means that it owns the "python3" namespace: diff --git a/macros.python2 b/macros.python2 index 0e4b0cb..df8e804 100644 --- a/macros.python2 +++ b/macros.python2 @@ -41,12 +41,12 @@ %py2_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python2_sitelib} - easy_install-%{python2_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} + %{__python2} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py2_install_wheel() %{expand:\\\ - pip%{python2_version} install -I dist/%{1} --root %{buildroot} --no-deps + %{__python2} -m pip install -I dist/%{1} --root %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python2_sitelib}/*.dist-info %{buildroot}%{python2_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/macros.python3 b/macros.python3 index 765e56b..691e035 100644 --- a/macros.python3 +++ b/macros.python3 @@ -30,17 +30,17 @@ %py3_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ + rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py3_install_egg() %{expand:\\\ mkdir -p %{buildroot}%{python3_sitelib} - easy_install-%{python3_version} -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} + %{__python3} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py3_install_wheel() %{expand:\\\ - pip%{python3_version} install -I dist/%{1} --root %{buildroot} --no-deps + %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --no-deps rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python3_sitelib}/*.dist-info %{buildroot}%{python3_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index eb381a8..5e5dac3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 5%{?dist} +Release: 6%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,11 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu Jul 16 2020 Miro Hrončok - 3.9-6 +- Make the unversioned %%__python macro error +- https://fedoraproject.org/wiki/Changes/PythonMacroError +- Make %%python macros more consistent with %%python3 macros + * Wed Jul 08 2020 Miro Hrončok - 3.9-5 - Introduce %%python_extras_subpkg - Adapt %%py_dist_name to keep square brackets diff --git a/tests/test_evals.py b/tests/test_evals.py index ce38793..917b355 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -362,3 +362,33 @@ def test_python_extras_subpkg_F(): It contains no code, just makes sure the dependencies are installed. """).lstrip().splitlines() assert lines == expected + + +unversioned_macros = pytest.mark.parametrize('macro', [ + '%__python', + '%python', + '%python_version', + '%python_version_nodots', + '%python_sitelib', + '%python_sitearch', + '%py_shebang_fix', + '%py_build', + '%py_build_egg', + '%py_build_wheel', + '%py_install', + '%py_install_egg', + '%py_install_wheel', +]) + + +@unversioned_macros +def test_unversioned_python_errors(macro): + lines = rpm_eval(macro, fails=True) + assert lines[0] == ('error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly') + + +@unversioned_macros +def test_unversioned_python_works_when_defined(macro): + macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') + assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) From 0086612c9853bd60ae73d9f6a89a9f2b0eb6fd31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 20 Jul 2020 17:33:32 +0200 Subject: [PATCH 106/218] Define %python_platform (as a Python version agnostic option to %python3_platform) %python2_platform is also defined, for consistency. --- macros.python | 1 + macros.python2 | 1 + python-rpm-macros.spec | 1 + tests/test_evals.py | 1 + 4 files changed, 4 insertions(+) diff --git a/macros.python b/macros.python index 2ecb49d..85c1896 100644 --- a/macros.python +++ b/macros.python @@ -4,6 +4,7 @@ %python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python2 b/macros.python2 index df8e804..201f2bd 100644 --- a/macros.python2 +++ b/macros.python2 @@ -2,6 +2,7 @@ %python2_sitearch %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python2_version %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python2_platform %(%{__python2} -Esc "import sysconfig; print(sysconfig.get_platform())") %py2_shbang_opts -s %py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5e5dac3..e476166 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -111,6 +111,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ - Make the unversioned %%__python macro error - https://fedoraproject.org/wiki/Changes/PythonMacroError - Make %%python macros more consistent with %%python3 macros +- Define %%python_platform (as a Python version agnostic option to %%python3_platform) * Wed Jul 08 2020 Miro Hrončok - 3.9-5 - Introduce %%python_extras_subpkg diff --git a/tests/test_evals.py b/tests/test_evals.py index 917b355..a46c6ad 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -371,6 +371,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_version_nodots', '%python_sitelib', '%python_sitearch', + '%python_platform', '%py_shebang_fix', '%py_build', '%py_build_egg', From 06ee3919932cc437f622c2c2d4971c27851b81b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 20 Jul 2020 17:36:12 +0200 Subject: [PATCH 107/218] Add --no-index --no-warn-script-location pip options to %pyX_install_wheel --no-index saves a ~1 minute timeout of pip possibly checking if there is a newer pip version (in mock with disabled network). --no-warn-script-location removes a bogus warning that says: WARNING: The scripts easy_install and easy_install-3.9 are installed in '/builddir/build/BUILDROOT/python-setuptools-49.2.0-1.fc33.x86_64/usr/bin' which is not on PATH. Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location. --- macros.python | 2 +- macros.python2 | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/macros.python b/macros.python index 85c1896..ddc737b 100644 --- a/macros.python +++ b/macros.python @@ -42,7 +42,7 @@ } %py_install_wheel() %{expand:\\\ - %{__python} -m pip install -I dist/%{1} --root %{buildroot} --no-deps + %{__python} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python_sitelib}/*.dist-info %{buildroot}%{python_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/macros.python2 b/macros.python2 index 201f2bd..2ebec9f 100644 --- a/macros.python2 +++ b/macros.python2 @@ -47,7 +47,7 @@ } %py2_install_wheel() %{expand:\\\ - %{__python2} -m pip install -I dist/%{1} --root %{buildroot} --no-deps + %{__python2} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python2_sitelib}/*.dist-info %{buildroot}%{python2_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/macros.python3 b/macros.python3 index 691e035..42e7143 100644 --- a/macros.python3 +++ b/macros.python3 @@ -40,7 +40,7 @@ } %py3_install_wheel() %{expand:\\\ - %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --no-deps + %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python3_sitelib}/*.dist-info %{buildroot}%{python3_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e476166..b0b2678 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -112,6 +112,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ - https://fedoraproject.org/wiki/Changes/PythonMacroError - Make %%python macros more consistent with %%python3 macros - Define %%python_platform (as a Python version agnostic option to %%python3_platform) +- Add --no-index --no-warn-script-location pip options to %%pyX_install_wheel * Wed Jul 08 2020 Miro Hrončok - 3.9-5 - Introduce %%python_extras_subpkg From b983c2118b85a734eae7e5f163611b9c4e90301e Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 16 Jul 2020 10:30:55 +0200 Subject: [PATCH 108/218] New opt-in possibility to fix byte-compilation reproducibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new script brp-fix-pyc-reproducibility creates an opt-in way of how to fix problems with the reproducibility of byte-compiled Python files. The script uses marshalparser [0] which currently doesn't provide solutions for all issues but can fix at least problems with reference flags. For more info see this Bugzilla [1]. If you want to use this new feature, you need to define `%py_reproducible_pyc_path` to specify a path you want to fix `.pyc` files in (recursively) and build-require /usr/bin/marshalparser. if you forget to build-require the parser. The error message is: ``` + /usr/lib/rpm/redhat/brp-python-bytecompile '' 1 0 Bytecompiling .py files below /builddir/build/BUILDROOT/tldr-0.5-2.fc33.x86_64/usr/lib/python3.9 using /usr/bin/python3.9 + /usr/lib/rpm/redhat/brp-fix-pyc-reproducibility /builddir/build/BUILDROOT/tldr-0.5-2.fc33.x86_64 ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser ! error: Bad exit status from /var/tmp/rpm-tmp.UUJr4v (%install) ``` A build fails if the parser is not able to parse any of the `.pyc` files. And finally, if a build is properly configured it produces fixed `.pyc` files. Currently, `.pyc` files in the tldr package contain a lot of unused reference flags: ``` $ dnf install -y tldr $ marshalparser --unused /usr/lib/python3.9/site-packages/__pycache__/tldr.cpython-39.pyc … long output … 190 - Flag_ref(byte=9610, type='TYPE_SHORT_ASCII_INTERNED', content=b'init', usages=0) 191 - Flag_ref(byte=9633, type='TYPE_SHORT_ASCII_INTERNED', content=b'source', usages=0) 192 - Flag_ref(byte=9651, type='TYPE_SHORT_ASCII_INTERNED', content=b'argv', usages=0) 193 - Flag_ref(byte=9657, type='TYPE_SHORT_ASCII_INTERNED', content=b'print_help', usages=0) 194 - Flag_ref(byte=9669, type='TYPE_SHORT_ASCII_INTERNED', content=b'stderr', usages=0) 195 - Flag_ref(byte=9682, type='TYPE_SHORT_ASCII_INTERNED', content=b'parse_args', usages=0) 196 - Flag_ref(byte=9737, type='TYPE_SHORT_ASCII_INTERNED', content=b'encode', usages=0) 197 - Flag_ref(byte=9782, type='TYPE_SHORT_ASCII_INTERNED', content=b'parser', usages=0) 198 - Flag_ref(byte=9790, type='TYPE_SHORT_ASCII_INTERNED', content=b'options', usages=0) 199 - Flag_ref(byte=9799, type='TYPE_SHORT_ASCII_INTERNED', content=b'rest', usages=0) 200 - Flag_ref(byte=9821, type='TYPE_SHORT_ASCII_INTERNED', content=b'result', usages=0) 202 - Flag_ref(byte=10022, type='TYPE_SHORT_ASCII_INTERNED', content=b'__main__', usages=0) 203 - Flag_ref(byte=10102, type='TYPE_SHORT_ASCII_INTERNED', content=b'argparse', usages=0) 204 - Flag_ref(byte=10433, type='TYPE_SHORT_ASCII_INTERNED', content=b'__name__', usages=0) 205 - Flag_ref(byte=10463, type='TYPE_SHORT_ASCII_INTERNED', content=b'', usages=0) ``` This new feature fixes them: ``` $ marshalparser --unused /usr/lib/python3.9/site-packages/__pycache__/tldr.cpython-39.pyc ``` [0] https://github.com/fedora-python/marshalparser [1] https://bugzilla.redhat.com/show_bug.cgi?id=1686078 --- brp-fix-pyc-reproducibility | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 brp-fix-pyc-reproducibility diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility new file mode 100644 index 0000000..4118d97 --- /dev/null +++ b/brp-fix-pyc-reproducibility @@ -0,0 +1,18 @@ +#!/bin/bash -e + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +# Defined as %py_reproducible_pyc_path macro and passed here as +# the first command-line argument +path_to_fix=$1 + +# First, check that the parser is available: +if [ ! -x /usr/bin/marshalparser ]; then + echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" + exit 1 +fi + +find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite From 638f809f4c15b8b14dfc1f6f8c3c08b0d2ba0b70 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 21 Jul 2020 12:59:36 +0200 Subject: [PATCH 109/218] Make %py3_dist respect %python3_pkgversion By default, %{py3_dist foo} generates python3dist(foo). This change makes it respect %python3_pkgversion so when it is redefined as X.Y, %{py3_dist foo} generates pythonX.Y(foo). --- macros.python-srpm | 3 ++- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index fa9f06a..494e3ff 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -90,6 +90,7 @@ # Needs to first put all arguments into a list, because invoking a different # macro (%py_dist_name) overwrites them %py3_dist() %{lua:\ + python3_pkgversion = rpm.expand("%python3_pkgversion");\ args = {}\ arg = 1\ while (true) do\ @@ -102,7 +103,7 @@ end\ for arg, name in ipairs(args) do\ canonical = rpm.expand("%py_dist_name " .. name);\ - print("python3dist(" .. canonical .. ") ");\ + print("python" .. python3_pkgversion .. "dist(" .. canonical .. ") ");\ end\ } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b0b2678..be4c08d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 6%{?dist} +Release: 7%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Jul 21 2020 Lumír Balhar - 3.9-7 +- Make %%py3_dist respect %%python3_pkgversion + * Thu Jul 16 2020 Miro Hrončok - 3.9-6 - Make the unversioned %%__python macro error - https://fedoraproject.org/wiki/Changes/PythonMacroError diff --git a/tests/test_evals.py b/tests/test_evals.py index a46c6ad..fa49439 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -50,6 +50,10 @@ def test_py3_dist(): assert rpm_eval(f'%py3_dist Aha[Boom] a') == ['python3dist(aha[boom]) python3dist(a)'] +def test_py3_dist_with_python3_pkgversion_redefined(): + assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion="3.6") == ['python3.6dist(aha[boom]) python3.6dist(a)'] + + def test_python_provide_python(): assert rpm_eval('%python_provide python-foo') == [] From 71c410dfa916461e5ded386b8585a028e72e3337 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 23 Jul 2020 08:09:55 +0200 Subject: [PATCH 110/218] Disable Python hash seed randomization in brp-python-bytecompile This change should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 --- brp-python-bytecompile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index c6afd1e..fbbd02b 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -114,6 +114,10 @@ fi # For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 # and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 +# Disable Python hash seed randomization +# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +export PYTHONHASHSEED=0 + shopt -s nullglob for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]+$"`; do From 0eae1d90da13934959153a0c291712ed369e2229 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 23 Jul 2020 12:23:49 +0200 Subject: [PATCH 111/218] Disable Python hash seed randomization in %py_byte_compile This change should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 --- macros.pybytecompile | 9 ++++++--- python-rpm-macros.spec | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index f34716b..7bd555b 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -12,9 +12,12 @@ # use it in a subshell like this: # (%{py_byte_compile }) || : +# Setting PYTHONHASHSEED=0 disables Python hash seed randomization +# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 + %py_byte_compile()\ py2_byte_compile () {\ - python_binary="%1"\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ failure=0\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ @@ -23,13 +26,13 @@ py2_byte_compile () {\ }\ \ py3_byte_compile () {\ - python_binary="%1"\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ - python_binary="%1"\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ }\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index be4c08d..8128055 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 7%{?dist} +Release: 8%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Fri Jul 24 2020 Lumír Balhar - 3.9-8 +- Disable Python hash seed randomization in %%py_byte_compile + * Tue Jul 21 2020 Lumír Balhar - 3.9-7 - Make %%py3_dist respect %%python3_pkgversion From 1979a78de9c2d96276130f723c0c70eb82a2b028 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Fri, 24 Jul 2020 07:50:20 +0200 Subject: [PATCH 112/218] Adapt %py(3)_shebang_fix to use versioned pathfixX.Y.py Versioned pathfixX.Y.py is available in main as well as in alternative Pythons so this change enables to build an alternative Python stack without a dependency on the main python3-devel. --- macros.python | 9 ++++++++- macros.python3 | 9 ++++++++- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 16 ++++++++-------- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/macros.python b/macros.python index ddc737b..1ed0219 100644 --- a/macros.python +++ b/macros.python @@ -10,7 +10,14 @@ %py_shbang_opts -s %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) -%py_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python} -k%{?py_shebang_flags:a %py_shebang_flags}} +%py_shebang_fix %{expand:\\\ + if [ -f /usr/bin/pathfix%{python_version}.py ]; then + pathfix=/usr/bin/pathfix%{python_version}.py + else + # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly + pathfix=/usr/bin/pathfix.py + fi + $pathfix -pni %{__python} -k%{?py_shebang_flags:a %py_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 42e7143..cfd8019 100644 --- a/macros.python3 +++ b/macros.python3 @@ -8,7 +8,14 @@ %py3_shbang_opts -s %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) -%py3_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python3} -k%{?py3_shebang_flags:a %py3_shebang_flags}} +%py3_shebang_fix %{expand:\\\ + if [ -f /usr/bin/pathfix%{python3_version}.py ]; then + pathfix=/usr/bin/pathfix%{python3_version}.py + else + # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly + pathfix=/usr/bin/pathfix.py + fi + $pathfix -pni %{__python3} -k%{?py3_shebang_flags:a %py3_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8128055..579fae1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 8%{?dist} +Release: 9%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Fri Jul 24 2020 Lumír Balhar - 3.9-9 +- Adapt %%py[3]_shebang_fix to use versioned pathfixX.Y.py + * Fri Jul 24 2020 Lumír Balhar - 3.9-8 - Disable Python hash seed randomization in %%py_byte_compile diff --git a/tests/test_evals.py b/tests/test_evals.py index fa49439..ec26668 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -228,23 +228,23 @@ def test_pypi_source_explicit_tilde(): def test_py3_shebang_fix(): - cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[0] - assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka s arg1 arg2 arg3' + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip() + assert cmd == '$pathfix -pni /usr/bin/python3 -ka s arg1 arg2 arg3' def test_py3_shebang_fix_custom_flags(): - cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags='Es')[0] - assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka Es arg1 arg2 arg3' + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags='Es')[-1].strip() + assert cmd == '$pathfix -pni /usr/bin/python3 -ka Es arg1 arg2 arg3' def test_py3_shebang_fix_empty_flags(): - cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags=None)[0] - assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -k arg1 arg2 arg3' + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags=None)[-1].strip() + assert cmd == '$pathfix -pni /usr/bin/python3 -k arg1 arg2 arg3' def test_py_shebang_fix_custom(): - cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[0] - assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy -ka s arg1 arg2 arg3' + cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip() + assert cmd == '$pathfix -pni /usr/bin/pypy -ka s arg1 arg2 arg3' def test_pycached_in_sitelib(): From 2eb41fe707d5fc22d3576ae5d1b1a6928ad56c67 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 23 Jul 2020 13:33:22 +0200 Subject: [PATCH 113/218] Implement an environment variable to run tests with specific macros --- tests/test_evals.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test_evals.py b/tests/test_evals.py index ec26668..5c8db71 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -8,9 +8,24 @@ import pytest X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' XY = f'{sys.version_info[0]}{sys.version_info[1]}' +# Handy environment variable you can use to run the tests +# with modified macros files. Multiple files should be +# separated by colon. +# To get 'em all, run: +# ls -1 macros.* | tr "\n" ":" +# and then: +# TESTED_FILES="" pytest -v +# or both combined: +# TESTED_FILES=$(ls -1 macros.* | tr "\n" ":") pytest -v +# Remember that some tests might need more macros files than just +# the local ones. +TESTED_FILES = os.getenv("TESTED_FILES", None) + def rpm_eval(expression, fails=False, **kwargs): cmd = ['rpmbuild'] + if TESTED_FILES: + cmd += ['--macros', TESTED_FILES] for var, value in kwargs.items(): if value is None: cmd += ['--undefine', var] From 431e4380cc8247bde656eacde1142ab9114be6d0 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Mon, 31 Aug 2020 13:44:41 +0200 Subject: [PATCH 114/218] Add a test for %py_byte_compile macro --- tests/pythontest.spec | 33 +++++++++++++++++++++++++++++++++ tests/tests.yml | 3 +++ 2 files changed, 36 insertions(+) create mode 100644 tests/pythontest.spec diff --git a/tests/pythontest.spec b/tests/pythontest.spec new file mode 100644 index 0000000..a0f052c --- /dev/null +++ b/tests/pythontest.spec @@ -0,0 +1,33 @@ +%global basedir /opt/test/byte_compilation + +Name: pythontest +Version: 0 +Release: 0 +Summary: ... +License: MIT +BuildRequires: python3-devel + +%description +... + +%install +mkdir -p %{buildroot}%{basedir}/directory/to/test/recursion + +echo "print()" > %{buildroot}%{basedir}/file.py +echo "print()" > %{buildroot}%{basedir}/directory/to/test/recursion/file_in_dir.py + +%py_byte_compile %{python3} %{buildroot}%{basedir}/file.py +%py_byte_compile %{python3} %{buildroot}%{basedir}/directory + +%check +# Count .py and .pyc files +PY=$(find %{buildroot}%{basedir} -name "*.py" | wc -l) +PYC=$(find %{buildroot}%{basedir} -name "*.pyc" | wc -l) + +# Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) +# so we should have two times more .pyc files than .py files +test $(expr $PY \* 2) -eq $PYC + +%files +%pycached %{basedir}/file.py +%pycached %{basedir}/directory/to/test/recursion/file_in_dir.py diff --git a/tests/tests.yml b/tests/tests.yml index 9fb4da3..7098b4b 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -16,6 +16,9 @@ - pytest: dir: . run: pytest -v + - manual_byte_compilation: + dir: . + run: rpmbuild -ba pythontest.spec required_packages: - rpm-build - python-rpm-macros From 02536540768555dde6d675935db45df81bcab7bf Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Mon, 7 Sep 2020 12:35:33 +0200 Subject: [PATCH 115/218] Use versioned pytest executable in `%pytest` macro for non-main Python stack --- macros.python3 | 2 +- tests/test_evals.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index cfd8019..714d9ca 100644 --- a/macros.python3 +++ b/macros.python3 @@ -73,7 +73,7 @@ } # This is intended for Python 3 only, hence also no Python version in the name. -%__pytest /usr/bin/pytest +%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) %pytest %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ diff --git a/tests/test_evals.py b/tests/test_evals.py index 5c8db71..20af7ca 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -188,6 +188,13 @@ def test_pytest_different_command(): assert 'pytest-3' in lines[-1] +def test_pytest_command_suffix(): + lines = rpm_eval('%pytest -v') + assert '/usr/bin/pytest -v' in lines[-1] + lines = rpm_eval('%pytest -v', python3_pkgversion="3.6", python3_version="3.6") + assert '/usr/bin/pytest-3.6 -v' in lines[-1] + + def test_pypi_source_default_name(): url = rpm_eval('%pypi_source', name='foo', version='6')[0] From a712d455f8db0de04c9beac6db2f93e213496265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 16 Sep 2020 17:32:10 +0200 Subject: [PATCH 116/218] python3-devel is required on the CI for pythontest.spec --- tests/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/tests.yml b/tests/tests.yml index 7098b4b..83968d2 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -23,5 +23,6 @@ - rpm-build - python-rpm-macros - python3-rpm-macros + - python3-devel - python3-pytest From 06987f5024e2618f3169ca829462b71f7bbae463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 14 Sep 2020 13:13:55 +0200 Subject: [PATCH 117/218] Add %python3_platform_triplet and %python3_ext_suffix Also add %python_platform_triplet and %python_ext_suffix. The CI tests are limited to x86_64 for now. https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names --- macros.python | 2 ++ macros.python3 | 2 ++ python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 17 +++++++++++++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 1ed0219..7059a8e 100644 --- a/macros.python +++ b/macros.python @@ -5,6 +5,8 @@ %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index 714d9ca..9e37a8a 100644 --- a/macros.python3 +++ b/macros.python3 @@ -3,6 +3,8 @@ %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 579fae1..e2c7092 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 9%{?dist} +Release: 10%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Sep 14 2020 Miro Hrončok - 3.9-10 +- Add %%python3_platform_triplet and %%python3_ext_suffix +- https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names + * Fri Jul 24 2020 Lumír Balhar - 3.9-9 - Adapt %%py[3]_shebang_fix to use versioned pathfixX.Y.py diff --git a/tests/test_evals.py b/tests/test_evals.py index 20af7ca..62bd39c 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,5 +1,6 @@ import os import subprocess +import platform import sys import textwrap @@ -398,6 +399,8 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_sitelib', '%python_sitearch', '%python_platform', + '%python_platform_triplet', + '%python_ext_suffix', '%py_shebang_fix', '%py_build', '%py_build_egg', @@ -419,3 +422,17 @@ def test_unversioned_python_errors(macro): def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + + +# we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 +x86_64_only = pytest.mark.skipif(platform.machine() != "x86_64", reason="works on x86_64 only") + + +@x86_64_only +def test_platform_triplet(): + assert rpm_eval("%python3_platform_triplet")[0] == "x86_64-linux-gnu" + + +@x86_64_only +def test_ext_suffix(): + assert rpm_eval("%python3_ext_suffix")[0] == f".cpython-{XY}-x86_64-linux-gnu.so" From a27bc6cc242e90f5daeaf6ee344d29c7192f97a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sun, 29 Nov 2020 17:44:37 +0100 Subject: [PATCH 118/218] BRP Python Bytecompile: Also detect Python files in /app/lib/pythonX.Y This is needed for flatpaks. Alternatively, we could pass %{_prefix} as an argument to this script, but that could make things a tad more complicated. This solution is less general, but more pragmatic. See https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/4FBBB3C5E63VDNGUJRLLW27LPZ74SEJH/ --- brp-python-bytecompile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index fbbd02b..012e8b5 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -119,7 +119,7 @@ fi export PYTHONHASHSEED=0 shopt -s nullglob -for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/usr/lib(64)?/python[0-9]\.[0-9]+$"`; +for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$"`; do python_binary=/usr/bin/$(basename $python_libdir) real_libdir=${python_libdir/$RPM_BUILD_ROOT/} From e5429a7a486e470b514aed207457fb237d061598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 8 Dec 2020 12:10:29 +0100 Subject: [PATCH 119/218] Support defining %py3_shebang_flags to %nil --- macros.python | 7 ++++++- macros.python2 | 8 +++++++- macros.python3 | 7 ++++++- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 34 +++++++++++++++++++++++++--------- 5 files changed, 48 insertions(+), 13 deletions(-) diff --git a/macros.python b/macros.python index 7059a8e..fb0d061 100644 --- a/macros.python +++ b/macros.python @@ -19,7 +19,12 @@ # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly pathfix=/usr/bin/pathfix.py fi - $pathfix -pni %{__python} -k%{?py_shebang_flags:a %py_shebang_flags}} + if [ -z "%{?py_shebang_flags}" ]; then + shebang_flags="-k" + else + shebang_flags="-ka%{py_shebang_flags}" + fi + $pathfix -pni %{__python} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 2ebec9f..d4d4eda 100644 --- a/macros.python2 +++ b/macros.python2 @@ -7,7 +7,13 @@ %py2_shbang_opts -s %py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) %py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-}) -%py2_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python2} -k%{?py2_shebang_flags:a %py2_shebang_flags}} +%py2_shebang_fix %{expand:\\\ + if [ -z "%{?py_shebang_flags}" ]; then + shebang_flags="-k" + else + shebang_flags="-ka%{py2_shebang_flags}" + fi + /usr/bin/pathfix.py -pni %{__python2} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 9e37a8a..323e675 100644 --- a/macros.python3 +++ b/macros.python3 @@ -17,7 +17,12 @@ # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly pathfix=/usr/bin/pathfix.py fi - $pathfix -pni %{__python3} -k%{?py3_shebang_flags:a %py3_shebang_flags}} + if [ -z "%{?py3_shebang_flags}" ]; then + shebang_flags="-k" + else + shebang_flags="-ka%{py3_shebang_flags}" + fi + $pathfix -pni %{__python3} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e2c7092..06a9a49 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 10%{?dist} +Release: 11%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Dec 08 2020 Miro Hrončok - 3.9-11 +- Support defining %%py3_shebang_flags to %%nil + * Mon Sep 14 2020 Miro Hrončok - 3.9-10 - Add %%python3_platform_triplet and %%python3_ext_suffix - https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names diff --git a/tests/test_evals.py b/tests/test_evals.py index 62bd39c..1de3a7b 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -42,6 +42,13 @@ def rpm_eval(expression, fails=False, **kwargs): return cp.stdout.strip().splitlines() +def shell_stdout(script): + return subprocess.check_output(script, + env={**os.environ, 'LANG': 'C.utf-8'}, + text=True, + shell=True).rstrip() + + @pytest.mark.parametrize('argument, result', [ ('a', 'a'), ('a-a', 'a-a'), @@ -252,22 +259,31 @@ def test_pypi_source_explicit_tilde(): def test_py3_shebang_fix(): cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 -ka s arg1 arg2 arg3' + assert cmd == '$pathfix -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' -def test_py3_shebang_fix_custom_flags(): - cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags='Es')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 -ka Es arg1 arg2 arg3' +def test_py3_shebang_fix_default_shebang_flags(): + lines = rpm_eval('%py3_shebang_fix arg1 arg2') + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-kas' -def test_py3_shebang_fix_empty_flags(): - cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags=None)[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 -k arg1 arg2 arg3' +def test_py3_shebang_fix_custom_shebang_flags(): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags='Es') + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-kaEs' -def test_py_shebang_fix_custom(): +@pytest.mark.parametrize('flags', [None, '%{nil}']) +def test_py3_shebang_fix_no_shebang_flags(flags): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-k' + + +def test_py_shebang_fix_custom_python(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/pypy -ka s arg1 arg2 arg3' + assert cmd == '$pathfix -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' def test_pycached_in_sitelib(): From 5b5cc39d89ceca9fba82f1c03cf06ea2f4e1f00b Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Wed, 27 Jan 2021 13:13:29 +0000 Subject: [PATCH 120/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 06a9a49..445de46 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 11%{?dist} +Release: 12%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Jan 27 2021 Fedora Release Engineering - 3.9-12 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild + * Tue Dec 08 2020 Miro Hrončok - 3.9-11 - Support defining %%py3_shebang_flags to %%nil From c746b25f28770ba2b4013fe94502f0a9f4659c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 5 Feb 2021 14:09:57 +0100 Subject: [PATCH 121/218] Automatically word-wrap the description of extras subpackages This only works for package and extra names less than 79 characters long. I don't expect many actual packages to exceed this limit. 78 characters should be enough for everybody. Why 78? 79 is the line-lenght limit from rpmlint. And we need to put punctuation in there. --- macros.python-srpm | 16 +++++++++++++--- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 40 ++++++++++++++++++++++++++++++++++------ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 494e3ff..f19ee45 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -202,9 +202,19 @@ local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname local summary = 'Summary: Metapackage for ' .. value_n .. ': ' .. extras .. ' extras' - local description = '%description -n ' .. rpmname .. '\\\n' .. - 'This is a metapackage bringing in ' .. extras .. ' extras requires for ' .. value_n .. '.\\\n' .. - 'It contains no code, just makes sure the dependencies are installed.\\\n' + local description = '%description -n ' .. rpmname .. '\\\n' + local current_line = 'This is a metapackage bringing in' + for _, word in ipairs({extras, 'extras', 'requires', 'for', value_n .. '.'}) do + local line = current_line .. ' ' .. word + if line:len() > 79 then + description = description .. current_line .. '\\\n' + current_line = word + else + current_line = line + end + end + description = description .. current_line .. '\\\n' .. + 'It contains no code, just makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 445de46..4e79fd2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 12%{?dist} +Release: 13%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -107,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Fri Feb 05 2021 Miro Hrončok - 3.9-13 +- Automatically word-wrap the description of extras subpackages +- Fixes: rhbz#1922442 + * Wed Jan 27 2021 Fedora Release Engineering - 3.9-12 - Rebuilt for https://fedoraproject.org/wiki/Fedora_34_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 1de3a7b..d460e3f 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -340,7 +340,8 @@ def test_python_extras_subpkg_i(): Summary: Metapackage for python3-setuptools_scm: toml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml @@ -350,7 +351,8 @@ def test_python_extras_subpkg_i(): Summary: Metapackage for python3-setuptools_scm: yaml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+yaml - This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + This is a metapackage bringing in yaml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml @@ -367,7 +369,8 @@ def test_python_extras_subpkg_f(): Summary: Metapackage for python3-setuptools_scm: toml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml -f ghost_filelist @@ -376,7 +379,8 @@ def test_python_extras_subpkg_f(): Summary: Metapackage for python3-setuptools_scm: yaml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+yaml - This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + This is a metapackage bringing in yaml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml -f ghost_filelist @@ -392,7 +396,8 @@ def test_python_extras_subpkg_F(): Summary: Metapackage for python3-setuptools_scm: toml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for python3-setuptools_scm. + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. @@ -401,12 +406,35 @@ def test_python_extras_subpkg_F(): Summary: Metapackage for python3-setuptools_scm: yaml extras Requires: python3-setuptools_scm = 6-7 %description -n python3-setuptools_scm+yaml - This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. + This is a metapackage bringing in yaml extras requires for + python3-setuptools_scm. It contains no code, just makes sure the dependencies are installed. """).lstrip().splitlines() assert lines == expected +@pytest.mark.parametrize('basename_len', [1, 10, 30, 45, 78]) +@pytest.mark.parametrize('extra_len', [1, 13, 28, 52, 78]) +def test_python_extras_subpkg_description_wrapping(basename_len, extra_len): + basename = 'x' * basename_len + extra = 'y' * extra_len + lines = rpm_eval(f'%python_extras_subpkg -n {basename} -F {extra}', + version='6', release='7') + for idx, line in enumerate(lines): + if line.startswith('%description'): + start = idx + 1 + lines = lines[start:] + assert all(len(l) < 80 for l in lines) + assert len(lines) < 6 + if len(" ".join(lines[:-1])) < 80: + assert len(lines) == 2 + expected_singleline = (f"This is a metapackage bringing in {extra} extras " + f"requires for {basename}. It contains no code, " + f"just makes sure the dependencies are installed.") + description_singleline = " ".join(lines) + assert description_singleline == expected_singleline + + unversioned_macros = pytest.mark.parametrize('macro', [ '%__python', '%python', From 230ce7f0613816bd7daf1f615bca443d6c5aa7d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 5 Feb 2021 14:56:23 +0100 Subject: [PATCH 122/218] Tests: Amend the comment for TESTED_FILES Arguably, this is easier. --- tests/test_evals.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/test_evals.py b/tests/test_evals.py index d460e3f..0c802c1 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -12,12 +12,8 @@ XY = f'{sys.version_info[0]}{sys.version_info[1]}' # Handy environment variable you can use to run the tests # with modified macros files. Multiple files should be # separated by colon. -# To get 'em all, run: -# ls -1 macros.* | tr "\n" ":" -# and then: -# TESTED_FILES="" pytest -v -# or both combined: -# TESTED_FILES=$(ls -1 macros.* | tr "\n" ":") pytest -v +# You can use * if you escape it from your Shell: +# TESTED_FILES='macros.*' pytest -v # Remember that some tests might need more macros files than just # the local ones. TESTED_FILES = os.getenv("TESTED_FILES", None) From 626168789c73d62eb211ec72fe260a128e66df45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 3 Feb 2021 21:40:23 +0100 Subject: [PATCH 123/218] Remove python2-rpm-macros https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros This is to be shipped together with an upgrade of python2.7: The python2.7 RPM package will contain the removed macros instead. The release is intentionally over-bumped to allow some changes of python-rpm-macros in lower versions of Fedora without the need to bump the version-release of python2-rpm-macros obsoleted by python2.7. --- macros.python2 | 64 ------------------------------------------ python-rpm-macros.spec | 23 ++++----------- 2 files changed, 5 insertions(+), 82 deletions(-) delete mode 100644 macros.python2 diff --git a/macros.python2 b/macros.python2 deleted file mode 100644 index d4d4eda..0000000 --- a/macros.python2 +++ /dev/null @@ -1,64 +0,0 @@ -%python2_sitelib %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python2_sitearch %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python2_version %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python2_platform %(%{__python2} -Esc "import sysconfig; print(sysconfig.get_platform())") - -%py2_shbang_opts -s -%py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) -%py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-}) -%py2_shebang_fix %{expand:\\\ - if [ -z "%{?py_shebang_flags}" ]; then - shebang_flags="-k" - else - shebang_flags="-ka%{py2_shebang_flags}" - fi - /usr/bin/pathfix.py -pni %{__python2} $shebang_flags} - -# Use the slashes after expand so that the command starts on the same line as -# the macro -# The `sleep 1` commands work around a race in install; see: -# https://bugzilla.redhat.com/show_bug.cgi?id=1644923 -%py2_build() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} - sleep 1 -} - -%py2_build_egg() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} - sleep 1 -} - -%py2_build_wheel() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} - sleep 1 -} - -%py2_install() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py2_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python2_sitelib} - %{__python2} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py2_install_wheel() %{expand:\\\ - %{__python2} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location - rm -rfv %{buildroot}%{_bindir}/__pycache__ - for distinfo in %{buildroot}%{python2_sitelib}/*.dist-info %{buildroot}%{python2_sitearch}/*.dist-info; do - if [ -f ${distinfo}/direct_url.json ]; then - rm -fv ${distinfo}/direct_url.json - sed -i '/direct_url.json/d' ${distinfo}/RECORD - fi - done -} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4e79fd2..0c4eb81 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 13%{?dist} +Release: 34%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -9,7 +9,6 @@ License: MIT and Python # Macros: Source101: macros.python Source102: macros.python-srpm -Source103: macros.python2 Source104: macros.python3 Source105: macros.pybytecompile @@ -48,19 +47,6 @@ Provides: bundled(python3dist(compileall2)) = %{compileall2_version} RPM macros for building Python source packages. -%package -n python2-rpm-macros -Summary: RPM macros for building Python 2 packages - -# For %%__python2 and %%python2 -Requires: python-srpm-macros = %{version}-%{release} - -# For %%py_setup -Requires: python-rpm-macros = %{version}-%{release} - -%description -n python2-rpm-macros -RPM macros for building Python 2 packages. - - %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages @@ -99,14 +85,15 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %{_rpmconfigdir}/redhat/compileall2.py %{_rpmluadir}/fedora/srpm/python.lua -%files -n python2-rpm-macros -%{rpmmacrodir}/macros.python2 - %files -n python3-rpm-macros %{rpmmacrodir}/macros.python3 %changelog +* Mon Feb 08 2021 Miro Hrončok - 3.9-34 +- Remove python2-rpm-macros +- https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros + * Fri Feb 05 2021 Miro Hrončok - 3.9-13 - Automatically word-wrap the description of extras subpackages - Fixes: rhbz#1922442 From a6382f5b5a6536d01477b71cf5a378c30c828783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 20 Feb 2021 12:52:02 +0100 Subject: [PATCH 124/218] Fix %python_extras_subpkg with underscores in extras names Fixes https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/FI6J7JNKIOYGBYIN5UJVWYG24UIIES2U/ --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 15 +++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f19ee45..c5935f9 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -198,7 +198,7 @@ rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' - for extras in args:gmatch('%w+') do + for extras in args:gmatch('%S+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname local summary = 'Summary: Metapackage for ' .. value_n .. ': ' .. extras .. ' extras' diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0c4eb81..27d568e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3.9 -Release: 34%{?dist} +Release: 35%{?dist} Summary: The common Python RPM macros # macros and lua: MIT, compileall2.py: PSFv2 @@ -90,6 +90,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Sat Feb 20 2021 Miro Hrončok - 3.9-35 +- Fix %%python_extras_subpkg with underscores in extras names + * Mon Feb 08 2021 Miro Hrončok - 3.9-34 - Remove python2-rpm-macros - https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros diff --git a/tests/test_evals.py b/tests/test_evals.py index 0c802c1..6904caa 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -409,6 +409,21 @@ def test_python_extras_subpkg_F(): assert lines == expected +def test_python_extras_subpkg_underscores(): + lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', + version='0.33.3', release='1.fc33') + expected = textwrap.dedent(f""" + %package -n python3-webscrapbook+adhoc_ssl + Summary: Metapackage for python3-webscrapbook: adhoc_ssl extras + Requires: python3-webscrapbook = 0.33.3-1.fc33 + %description -n python3-webscrapbook+adhoc_ssl + This is a metapackage bringing in adhoc_ssl extras requires for + python3-webscrapbook. + It contains no code, just makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + @pytest.mark.parametrize('basename_len', [1, 10, 30, 45, 78]) @pytest.mark.parametrize('extra_len', [1, 13, 28, 52, 78]) def test_python_extras_subpkg_description_wrapping(basename_len, extra_len): From 8a1e9e09530d2e13a72c9f9808d64985054e0463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 23 Feb 2021 23:21:47 +0100 Subject: [PATCH 125/218] Set Version: %{__default_python3_version} to remove one place to bump Since Fedora 33, the package version always == %{__default_python3_version}. When we update Python to 3.X+1, we update the version and the macro. When the macro is not updated confusing things happen. See for example https://bugzilla.redhat.com/show_bug.cgi?id=1931421#c4 We could assert the versions match in %check instead, but this way a temporary pull request for a new Python version, such as https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/50 https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/84 can be kept rebased via the git forge UI. Note that the value of %{__default_python3_version} is loaded from sources in dist git, otherwise it would be defined by the previous build of this package. As a result, the spec file is no longer parsable on it's own, but IMHO that's OK. --- python-rpm-macros.spec | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 27d568e..cd2437b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,11 +1,6 @@ Name: python-rpm-macros -Version: 3.9 -Release: 35%{?dist} Summary: The common Python RPM macros -# macros and lua: MIT, compileall2.py: PSFv2 -License: MIT and Python - # Macros: Source101: macros.python Source102: macros.python-srpm @@ -19,6 +14,16 @@ Source201: python.lua %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py +# macros and lua: MIT, compileall2.py: PSFv2 +License: MIT and Python + +# The package version MUST be always the same as %%{__default_python3_version}. +# To have only one source of truth, we load the macro and use it. +# The macro is defined in python-srpm-macros. + %{load:%{SOURCE102}} +Version: %{__default_python3_version} +Release: 35%{?dist} + BuildArch: noarch # For %%__default_python3_pkgversion used in %%python_provide From 1edfea6956eb2c9e12e4e06e7dbac4cc8fb202b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 24 Feb 2021 13:06:35 +0100 Subject: [PATCH 126/218] Update comment for python_altnames() to reflect the reality --- python.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python.lua b/python.lua index 8766c91..8391fe7 100644 --- a/python.lua +++ b/python.lua @@ -3,8 +3,8 @@ -- Determine alternate names provided from the given name. -- Used in pythonname provides generator, python_provide and py_provides. -- There are 2 rules: --- python3-foo -> python-foo, python3X-foo --- python3X-foo -> python-foo, python3-foo +-- python3-foo -> python-foo, python3.X-foo +-- python3.X-foo -> python-foo, python3-foo -- There is no python-foo -> rule, python-foo packages are version agnostic. -- Returns a table/array with strings. Empty when no rule matched. local function python_altnames(name) From 4805d44fa0dc4adfdc66b50cbdd5f57fd73ad8e6 Mon Sep 17 00:00:00 2001 From: Kalev Lember Date: Wed, 10 Mar 2021 17:09:14 +0100 Subject: [PATCH 127/218] BRP Python Bytecompile: Avoid hardcoding /usr/bin prefix for python Avoid using the full path and instead rely on PATH being correctly set up to find the executable. This fixes byte compilation for python2.7 when doing flatpak module builds where python2.7 can be in either /usr/bin or /app/bin, depending on how it's compiled. --- brp-python-bytecompile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 012e8b5..ead7022 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -121,7 +121,7 @@ export PYTHONHASHSEED=0 shopt -s nullglob for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$"`; do - python_binary=/usr/bin/$(basename $python_libdir) + python_binary=$(basename $python_libdir) real_libdir=${python_libdir/$RPM_BUILD_ROOT/} echo "Bytecompiling .py files below $python_libdir using $python_binary" From bc016cbbc532e878c1357a160d1a02d1902d9c6d Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 16 Mar 2021 08:45:18 +0100 Subject: [PATCH 128/218] Make extras_subpkg description more general Because extra subpackages actually might contain code. See for example: https://src.fedoraproject.org/rpms/python-dns/pull-request/9 --- macros.python-srpm | 2 +- tests/test_evals.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index c5935f9..996b0f9 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -214,7 +214,7 @@ end end description = description .. current_line .. '\\\n' .. - 'It contains no code, just makes sure the dependencies are installed.\\\n' + 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i diff --git a/tests/test_evals.py b/tests/test_evals.py index 6904caa..1f931c6 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -338,7 +338,7 @@ def test_python_extras_subpkg_i(): %description -n python3-setuptools_scm+toml This is a metapackage bringing in toml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info @@ -349,7 +349,7 @@ def test_python_extras_subpkg_i(): %description -n python3-setuptools_scm+yaml This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info @@ -367,7 +367,7 @@ def test_python_extras_subpkg_f(): %description -n python3-setuptools_scm+toml This is a metapackage bringing in toml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml -f ghost_filelist @@ -377,7 +377,7 @@ def test_python_extras_subpkg_f(): %description -n python3-setuptools_scm+yaml This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml -f ghost_filelist """).lstrip().splitlines() @@ -394,7 +394,7 @@ def test_python_extras_subpkg_F(): %description -n python3-setuptools_scm+toml This is a metapackage bringing in toml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. @@ -404,7 +404,7 @@ def test_python_extras_subpkg_F(): %description -n python3-setuptools_scm+yaml This is a metapackage bringing in yaml extras requires for python3-setuptools_scm. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. """).lstrip().splitlines() assert lines == expected @@ -419,7 +419,7 @@ def test_python_extras_subpkg_underscores(): %description -n python3-webscrapbook+adhoc_ssl This is a metapackage bringing in adhoc_ssl extras requires for python3-webscrapbook. - It contains no code, just makes sure the dependencies are installed. + It makes sure the dependencies are installed. """).lstrip().splitlines() assert lines == expected @@ -440,8 +440,8 @@ def test_python_extras_subpkg_description_wrapping(basename_len, extra_len): if len(" ".join(lines[:-1])) < 80: assert len(lines) == 2 expected_singleline = (f"This is a metapackage bringing in {extra} extras " - f"requires for {basename}. It contains no code, " - f"just makes sure the dependencies are installed.") + f"requires for {basename}. " + f"It makes sure the dependencies are installed.") description_singleline = " ".join(lines) assert description_singleline == expected_singleline From a44ae31ad064f6cfeeb1309bc7093db2e66e6e43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 29 Mar 2021 14:49:20 +0200 Subject: [PATCH 129/218] Allow commas as argument separator for extras names in %python_extras_subpkg This allows e.g.: %global extras cli,ghostwriter,pytz,dateutil,lark,numpy,pandas,pytest,redis,zoneinfo,django %{pyproject_extras_subpkg -n python3-hypothesis %{extras}} ... %pyproject_buildrequires -x %{extras} (Note that %pyproject_extras_subpkg is a tiny wrapper around %python_extras_subpkg.) --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 41 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 996b0f9..efe2bba 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -198,7 +198,7 @@ rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' - for extras in args:gmatch('%S+') do + for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname local summary = 'Summary: Metapackage for ' .. value_n .. ': ' .. extras .. ' extras' diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index cd2437b..e445285 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -22,7 +22,7 @@ License: MIT and Python # The macro is defined in python-srpm-macros. %{load:%{SOURCE102}} Version: %{__default_python3_version} -Release: 35%{?dist} +Release: 36%{?dist} BuildArch: noarch @@ -95,6 +95,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Mar 29 2021 Miro Hrončok - 3.9-36 +- Allow commas as argument separator for extras names in %%python_extras_subpkg +- Fixes: rhbz#1936486 + * Sat Feb 20 2021 Miro Hrončok - 3.9-35 - Fix %%python_extras_subpkg with underscores in extras names diff --git a/tests/test_evals.py b/tests/test_evals.py index 1f931c6..e2909c3 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -424,6 +424,47 @@ def test_python_extras_subpkg_underscores(): assert lines == expected +@pytest.mark.parametrize('sep', [pytest.param(('', ' ', ' ', ''), id='spaces'), + pytest.param(('', ',', ',', ''), id='commas'), + pytest.param(('', ',', ',', ','), id='commas-trailing'), + pytest.param((',', ',', ',', ''), id='commas-leading'), + pytest.param((',', ',', ',', ','), id='commas-trailing-leading'), + pytest.param(('', ',', ' ', ''), id='mixture'), + pytest.param((' ', ' ', '\t\t, ', '\t'), id='chaotic-good'), + pytest.param(('', '\t ,, \t\r ', ',,\t , ', ',,'), id='chaotic-evil')]) +def test_python_extras_subpkg_arg_separators(sep): + lines = rpm_eval('%python_extras_subpkg -n python3-hypothesis -F {}cli{}ghostwriter{}pytz{}'.format(*sep), + version='6.6.0', release='1.fc35') + expected = textwrap.dedent(f""" + %package -n python3-hypothesis+cli + Summary: Metapackage for python3-hypothesis: cli extras + Requires: python3-hypothesis = 6.6.0-1.fc35 + %description -n python3-hypothesis+cli + This is a metapackage bringing in cli extras requires for python3-hypothesis. + It makes sure the dependencies are installed. + + + + %package -n python3-hypothesis+ghostwriter + Summary: Metapackage for python3-hypothesis: ghostwriter extras + Requires: python3-hypothesis = 6.6.0-1.fc35 + %description -n python3-hypothesis+ghostwriter + This is a metapackage bringing in ghostwriter extras requires for + python3-hypothesis. + It makes sure the dependencies are installed. + + + + %package -n python3-hypothesis+pytz + Summary: Metapackage for python3-hypothesis: pytz extras + Requires: python3-hypothesis = 6.6.0-1.fc35 + %description -n python3-hypothesis+pytz + This is a metapackage bringing in pytz extras requires for python3-hypothesis. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + @pytest.mark.parametrize('basename_len', [1, 10, 30, 45, 78]) @pytest.mark.parametrize('extra_len', [1, 13, 28, 52, 78]) def test_python_extras_subpkg_description_wrapping(basename_len, extra_len): From dcb4422895348422914d25c328486c6e30ff61dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 29 Mar 2021 15:49:29 +0200 Subject: [PATCH 130/218] Escape a macro in an old %changelog entry --- python-rpm-macros.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e445285..abcab86 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -237,7 +237,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ - Add %%python_disable_dependency_generator * Wed Dec 05 2018 Miro Hrončok - 3-40 -- Workaround leaking buildroot PATH in %py_byte_compile (#1647212) +- Workaround leaking buildroot PATH in %%py_byte_compile (#1647212) * Thu Nov 01 2018 Petr Viktorin - 3-39 - Move "sleep 1" workaround from py3_build to py2_build (#1644923) From bd4c3de20c94286372a32b1bb953fbb539e95b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 29 Mar 2021 15:52:12 +0200 Subject: [PATCH 131/218] Make the spec file parseable when %_sourcedir is not . This happens e.g. with: - rpmlint python-rpm-macros.spec - rpmdev-bumpspec python-rpm-macros.spec --- python-rpm-macros.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index abcab86..20ec1d7 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -20,7 +20,7 @@ License: MIT and Python # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. # The macro is defined in python-srpm-macros. - %{load:%{SOURCE102}} + %{?load:%{SOURCE102}} Version: %{__default_python3_version} Release: 36%{?dist} From 9d2fcef337254a29a4635c62dcec725cdab836d7 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 7 Apr 2021 16:48:04 +0200 Subject: [PATCH 132/218] Use sysconfig.get_path() to define %python_sitelib and %python_sitearch Distutils which were used to define the macros are deprecated in Python3.10: https://www.python.org/dev/peps/pep-0632/. Sysconfig isn't and it works across our Pythons, making it better choice for the task. --- macros.python | 4 ++-- macros.python3 | 4 ++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 38 ++++++++++++++++++++++++++++++++++++-- tests/tests.yml | 1 + 5 files changed, 46 insertions(+), 7 deletions(-) diff --git a/macros.python b/macros.python index fb0d061..79ff2d4 100644 --- a/macros.python +++ b/macros.python @@ -1,7 +1,7 @@ # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros -%python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") +%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") diff --git a/macros.python3 b/macros.python3 index 323e675..480da96 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,5 +1,5 @@ -%python3_sitelib %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") +%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") +%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 20ec1d7..9094b3e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -22,7 +22,7 @@ License: MIT and Python # The macro is defined in python-srpm-macros. %{?load:%{SOURCE102}} Version: %{__default_python3_version} -Release: 36%{?dist} +Release: 37%{?dist} BuildArch: noarch @@ -95,6 +95,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Apr 07 2021 Karolina Surma - 3.9-37 +- Use sysconfig.get_path() to get %%python3_sitelib and %%python3_sitearch +- Fixes: rhbz#1946972 + * Mon Mar 29 2021 Miro Hrončok - 3.9-36 - Allow commas as argument separator for extras names in %%python_extras_subpkg - Fixes: rhbz#1936486 diff --git a/tests/test_evals.py b/tests/test_evals.py index e2909c3..b5af05f 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -38,6 +38,17 @@ def rpm_eval(expression, fails=False, **kwargs): return cp.stdout.strip().splitlines() +@pytest.fixture(scope="session") +def lib(): + lib_eval = rpm_eval("%_lib")[0] + if lib_eval == "%_lib" and TESTED_FILES: + raise ValueError( + "%_lib is not resolved to an actual value. " + "You may want to include /usr/lib/rpm/platform/x86_64-linux/macros to TESTED_FILES." + ) + return lib_eval + + def shell_stdout(script): return subprocess.check_output(script, env={**os.environ, 'LANG': 'C.utf-8'}, @@ -290,9 +301,8 @@ def test_pycached_in_sitelib(): ] -def test_pycached_in_sitearch(): +def test_pycached_in_sitearch(lib): lines = rpm_eval('%pycached %{python3_sitearch}/foo*.py') - lib = rpm_eval('%_lib')[0] assert lines == [ f'/usr/{lib}/python{X_Y}/site-packages/foo*.py', f'/usr/{lib}/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' @@ -532,3 +542,27 @@ def test_platform_triplet(): @x86_64_only def test_ext_suffix(): assert rpm_eval("%python3_ext_suffix")[0] == f".cpython-{XY}-x86_64-linux-gnu.so" + + +def test_python_sitelib_value(): + macro = '%python_sitelib' + assert rpm_eval(macro, __python='/usr/bin/python3.6')[0] == f'/usr/lib/python3.6/site-packages' + assert rpm_eval(macro, __python='%__python3')[0] == f'/usr/lib/python{X_Y}/site-packages' + + +def test_python3_sitelib_value(): + macro = '%python3_sitelib' + assert rpm_eval(macro, __python3='/usr/bin/python3.6')[0] == f'/usr/lib/python3.6/site-packages' + assert rpm_eval(macro)[0] == f'/usr/lib/python{X_Y}/site-packages' + + +def test_python_sitearch_value(lib): + macro = '%python_sitearch' + assert rpm_eval(macro, __python='/usr/bin/python3.6')[0] == f'/usr/{lib}/python3.6/site-packages' + assert rpm_eval(macro, __python='%__python3')[0] == f'/usr/{lib}/python{X_Y}/site-packages' + + +def test_python3_sitearch_value(lib): + macro = '%python3_sitearch' + assert rpm_eval(macro, __python3='/usr/bin/python3.6')[0] == f'/usr/{lib}/python3.6/site-packages' + assert rpm_eval(macro)[0] == f'/usr/{lib}/python{X_Y}/site-packages' diff --git a/tests/tests.yml b/tests/tests.yml index 83968d2..d6090ec 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -25,4 +25,5 @@ - python3-rpm-macros - python3-devel - python3-pytest + - python3.6 From 39166a7b4b5f0d52480c01a124d6ec514b2e0f71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 8 Apr 2021 15:33:22 +0200 Subject: [PATCH 133/218] Tests: Assert single-line macros are single-line --- tests/test_evals.py | 76 ++++++++++++++++++++++----------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/test_evals.py b/tests/test_evals.py index b5af05f..6a67e32 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -211,57 +211,57 @@ def test_pytest_command_suffix(): def test_pypi_source_default_name(): - url = rpm_eval('%pypi_source', - name='foo', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + urls = rpm_eval('%pypi_source', + name='foo', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'] def test_pypi_source_default_srcname(): - url = rpm_eval('%pypi_source', - name='python-foo', srcname='foo', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + urls = rpm_eval('%pypi_source', + name='python-foo', srcname='foo', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'] def test_pypi_source_default_pypi_name(): - url = rpm_eval('%pypi_source', - name='python-foo', pypi_name='foo', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + urls = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'] def test_pypi_source_default_name_uppercase(): - url = rpm_eval('%pypi_source', - name='Foo', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz' + urls = rpm_eval('%pypi_source', + name='Foo', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz'] def test_pypi_source_provided_name(): - url = rpm_eval('%pypi_source foo', - name='python-bar', pypi_name='bar', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + urls = rpm_eval('%pypi_source foo', + name='python-bar', pypi_name='bar', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'] def test_pypi_source_provided_name_version(): - url = rpm_eval('%pypi_source foo 6', - name='python-bar', pypi_name='bar', version='3')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + urls = rpm_eval('%pypi_source foo 6', + name='python-bar', pypi_name='bar', version='3') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz'] def test_pypi_source_provided_name_version_ext(): url = rpm_eval('%pypi_source foo 6 zip', - name='python-bar', pypi_name='bar', version='3')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip' + name='python-bar', pypi_name='bar', version='3') + assert url == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip'] def test_pypi_source_prerelease(): - url = rpm_eval('%pypi_source', - name='python-foo', pypi_name='foo', version='6~b2')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz' + urls = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6~b2') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz'] def test_pypi_source_explicit_tilde(): - url = rpm_eval('%pypi_source foo 6~6', - name='python-foo', pypi_name='foo', version='6')[0] - assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz' + urls = rpm_eval('%pypi_source foo 6~6', + name='python-foo', pypi_name='foo', version='6') + assert urls == ['https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz'] def test_py3_shebang_fix(): @@ -520,8 +520,8 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) - assert lines[0] == ('error: attempt to use unversioned python, ' - 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly') + assert lines == ['error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'] @unversioned_macros @@ -536,33 +536,33 @@ x86_64_only = pytest.mark.skipif(platform.machine() != "x86_64", reason="works o @x86_64_only def test_platform_triplet(): - assert rpm_eval("%python3_platform_triplet")[0] == "x86_64-linux-gnu" + assert rpm_eval("%python3_platform_triplet") == ["x86_64-linux-gnu"] @x86_64_only def test_ext_suffix(): - assert rpm_eval("%python3_ext_suffix")[0] == f".cpython-{XY}-x86_64-linux-gnu.so" + assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"] def test_python_sitelib_value(): macro = '%python_sitelib' - assert rpm_eval(macro, __python='/usr/bin/python3.6')[0] == f'/usr/lib/python3.6/site-packages' - assert rpm_eval(macro, __python='%__python3')[0] == f'/usr/lib/python{X_Y}/site-packages' + assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] + assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages'] def test_python3_sitelib_value(): macro = '%python3_sitelib' - assert rpm_eval(macro, __python3='/usr/bin/python3.6')[0] == f'/usr/lib/python3.6/site-packages' - assert rpm_eval(macro)[0] == f'/usr/lib/python{X_Y}/site-packages' + assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] + assert rpm_eval(macro) == [f'/usr/lib/python{X_Y}/site-packages'] def test_python_sitearch_value(lib): macro = '%python_sitearch' - assert rpm_eval(macro, __python='/usr/bin/python3.6')[0] == f'/usr/{lib}/python3.6/site-packages' - assert rpm_eval(macro, __python='%__python3')[0] == f'/usr/{lib}/python{X_Y}/site-packages' + assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] + assert rpm_eval(macro, __python='%__python3') == [f'/usr/{lib}/python{X_Y}/site-packages'] def test_python3_sitearch_value(lib): macro = '%python3_sitearch' - assert rpm_eval(macro, __python3='/usr/bin/python3.6')[0] == f'/usr/{lib}/python3.6/site-packages' - assert rpm_eval(macro)[0] == f'/usr/{lib}/python{X_Y}/site-packages' + assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] + assert rpm_eval(macro) == [f'/usr/{lib}/python{X_Y}/site-packages'] From 187e049d6cf796b692e608043f781c0e0b98b95f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 8 Apr 2021 15:39:13 +0200 Subject: [PATCH 134/218] Document a TESTED_FILES value that currently works --- tests/test_evals.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_evals.py b/tests/test_evals.py index 6a67e32..43216ca 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -15,7 +15,8 @@ XY = f'{sys.version_info[0]}{sys.version_info[1]}' # You can use * if you escape it from your Shell: # TESTED_FILES='macros.*' pytest -v # Remember that some tests might need more macros files than just -# the local ones. +# the local ones. You might need to use: +# TESTED_FILES='/usr/lib/rpm/macros:/usr/lib/rpm/platform/x86_64-linux/macros:macros.*' TESTED_FILES = os.getenv("TESTED_FILES", None) From cad73c2159567a9857783c56f6b88fc8346a5df9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 27 Apr 2021 10:50:50 +0200 Subject: [PATCH 135/218] Be more careful when loading the macros from sources The %{?load:%{SOURCE102}} construct no longer works in RPM 4.17+ Currently, we: 1. Load %{SOURCE102} if it exists. This should always be the case when actually building the RPM or SRPM package. 2. Else, load macros.python-srpm if it exists. This is the case when something parses the spec from dist-git without setting %_sourcedir to the current working directory. E.g. rpmdev-bumpspec does this. 3. Else, don't load anything, get %{__default_python3_version} from the environment. This is the case when something parses the spec in isolation. Getting the version from sources is impossible, because the sources are missing. So we get the installed version instead. Note that this will blow up on Fedora < 33, but it already did before. --- python-rpm-macros.spec | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 9094b3e..2a5f6fa 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -20,7 +20,14 @@ License: MIT and Python # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. # The macro is defined in python-srpm-macros. - %{?load:%{SOURCE102}} +%{lua: +if posix.stat(rpm.expand('%{SOURCE102}')) then + rpm.load(rpm.expand('%{SOURCE102}')) +elseif posix.stat('macros.python-srpm') then + -- something is parsing the spec without _sourcedir macro properly set + rpm.load('macros.python-srpm') +end +} Version: %{__default_python3_version} Release: 37%{?dist} From 03a1e3ba65feeb40adc3261f0bd0b0112d1a387b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 27 Apr 2021 10:00:02 +0200 Subject: [PATCH 136/218] Escape % symbols in macro files comments This is most likely not neccessary but can prevent serious problems like: https://bugzilla.redhat.com/show_bug.cgi?id=1953910 --- macros.pybytecompile | 6 +++--- macros.python-srpm | 14 +++++++------- python-rpm-macros.spec | 11 ++++++++++- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index 7bd555b..b58fff4 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -4,13 +4,13 @@ # Which unfortunately makes the definition more complicated than it should be # Usage: -# %py_byte_compile +# %%py_byte_compile # Example: -# %py_byte_compile %{__python3} %{buildroot}%{_datadir}/spam/plugins/ +# %%py_byte_compile %%{__python3} %%{buildroot}%%{_datadir}/spam/plugins/ # This will terminate build on SyntaxErrors, if you want to avoid that, # use it in a subshell like this: -# (%{py_byte_compile }) || : +# (%%{py_byte_compile }) || : # Setting PYTHONHASHSEED=0 disables Python hash seed randomization # This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 diff --git a/macros.python-srpm b/macros.python-srpm index efe2bba..4957492 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -47,7 +47,7 @@ # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 -# Set to /bin/true to avoid %ifdefs and %{? in specfiles +# Set to /bin/true to avoid %%ifdefs and %%{? in specfiles %__python3_other /bin/true %py3_other_build /bin/true %py3_other_install /bin/true @@ -68,7 +68,7 @@ # Creates Python 2 dist tag(s) after converting names to canonical format # Needs to first put all arguments into a list, because invoking a different -# macro (%py_dist_name) overwrites them +# macro (%%py_dist_name) overwrites them %py2_dist() %{lua:\ args = {}\ arg = 1\ @@ -88,7 +88,7 @@ # Creates Python 3 dist tag(s) after converting names to canonical format # Needs to first put all arguments into a list, because invoking a different -# macro (%py_dist_name) overwrites them +# macro (%%py_dist_name) overwrites them %py3_dist() %{lua:\ python3_pkgversion = rpm.expand("%python3_pkgversion");\ args = {}\ @@ -110,12 +110,12 @@ # Macro to replace overly complicated references to PyPI source files. # Expands to the pythonhosted URL for a package # Accepts zero to three arguments: -# 1: The PyPI project name, defaulting to %srcname if it is defined, then -# %pypi_name if it is defined, then just %name. -# 2: The PYPI version, defaulting to %version with tildes stripped. +# 1: The PyPI project name, defaulting to %%srcname if it is defined, then +# %%pypi_name if it is defined, then just %%name. +# 2: The PYPI version, defaulting to %%version with tildes stripped. # 3: The file extension, defaulting to "tar.gz". (A period will be added # automatically.) -# Requires %__pypi_url and %__pypi_default_extension to be defined. +# Requires %%__pypi_url and %%__pypi_default_extension to be defined. %__pypi_url https://files.pythonhosted.org/packages/source/ %__pypi_default_extension tar.gz diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 2a5f6fa..55681ec 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -29,7 +29,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 37%{?dist} +Release: 38%{?dist} BuildArch: noarch @@ -88,6 +88,11 @@ mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +%check +# no macros in comments +! grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* + + %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile @@ -102,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Apr 27 2021 Miro Hrončok - 3.9-38 +- Escape %% symbols in macro files comments +- Fixes: rhbz#1953910 + * Wed Apr 07 2021 Karolina Surma - 3.9-37 - Use sysconfig.get_path() to get %%python3_sitelib and %%python3_sitearch - Fixes: rhbz#1946972 From 14e4c04a425bb1951848712c16e0e10c50343df9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 27 Apr 2021 15:49:21 +0200 Subject: [PATCH 137/218] Remove EPEL 7 compatibility macros that were actually not defined at all The %{? in the comment made the entire block of macros not work. Since nobody actually used those on Fedora, because they did not exist, we can safely remove them. No need to document this in the %changelog. --- macros.python-srpm | 7 ------- 1 file changed, 7 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 4957492..9a21c23 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -47,13 +47,6 @@ # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 -# Set to /bin/true to avoid %%ifdefs and %%{? in specfiles -%__python3_other /bin/true -%py3_other_build /bin/true -%py3_other_install /bin/true - - - # === Macros for Build/Requires tags using Python dist tags === # - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages # - These macros need to be in macros.python-srpm, because BuildRequires tags From 2b43f896affae70b94af42c9676a334d7bc9502f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 27 Apr 2021 15:53:13 +0200 Subject: [PATCH 138/218] Update %python3_pkgversion comment --- macros.python-srpm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 9a21c23..f6f1db6 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -43,8 +43,10 @@ # This is left intentionally a separate macro, in case the naming convention ever changes. %__default_python3_pkgversion %__default_python3_version -# python3_pkgversion specifies the version of Python 3 in the distro. It can be -# a specific version (e.g. 34 in Fedora EPEL7) +# python3_pkgversion specifies the version of Python 3 in the distro. +# For Fedora, this is usually just "3". +# It can be a specific version distro-wide (e.g. "36" in EPEL7). +# Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 # === Macros for Build/Requires tags using Python dist tags === From 3a654e3bed3f73c530c49fcf21d8d323614baff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 1 Jun 2021 15:57:22 +0200 Subject: [PATCH 139/218] Python 3.10 https://fedoraproject.org/wiki/Changes/Python3.10 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f6f1db6..844dfeb 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -36,7 +36,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.9 +%__default_python3_version 3.10 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 55681ec..c3e5e19 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -29,7 +29,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 38%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -107,6 +107,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Jun 01 2021 Miro Hrončok - 3.10-1 +- Update main Python to Python 3.10 +- https://fedoraproject.org/wiki/Changes/Python3.10 + * Tue Apr 27 2021 Miro Hrončok - 3.9-38 - Escape %% symbols in macro files comments - Fixes: rhbz#1953910 From 370b825e45fe4f592640f3c935976ca65b596aca Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 3 Jun 2021 11:44:21 +0200 Subject: [PATCH 140/218] Add the project's canonical URL --- python-rpm-macros.spec | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c3e5e19..dd9cc93 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,8 @@ Name: python-rpm-macros Summary: The common Python RPM macros +URL: https://src.fedoraproject.org/rpms/python-rpm-macros/ + # Macros: Source101: macros.python Source102: macros.python-srpm From 9dff7fbf6a3184cdbc550ab28623229bb8c5085b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 15 Jun 2021 16:15:45 +0200 Subject: [PATCH 141/218] Fix %python_provide when fed python3.10-foo to obsolete python-foo instead of python--foo This has unlikely broken anything in practice, no packages in Fedora use %python_provide with major.minor-version-prefixed names. --- macros.python | 5 +++-- python-rpm-macros.spec | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/macros.python b/macros.python index 79ff2d4..f0c3f14 100644 --- a/macros.python +++ b/macros.python @@ -74,6 +74,7 @@ local package = rpm.expand("%{?1}") local vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") local provides = python.python_altprovides(package, vr) + local default_python3_pkgversion = rpm.expand("%{__default_python3_pkgversion}") if (string.starts(package, "python3-")) then for i, provide in ipairs(provides) do print("\\nProvides: " .. provide) @@ -84,14 +85,14 @@ print(string.sub(package,9,string.len(package))) print(" < " .. vr) end - elseif (string.starts(package, "python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-")) then + elseif (string.starts(package, "python" .. default_python3_pkgversion .. "-")) then for i, provide in ipairs(provides) do print("\\nProvides: " .. provide) end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") - print(string.sub(package,11,string.len(package))) + print(string.sub(package,8+string.len(default_python3_pkgversion),string.len(package))) print(" < " .. vr) end elseif (string.starts(package, "python")) then diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index dd9cc93..04fe84c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -31,7 +31,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -109,6 +109,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Jun 15 2021 Miro Hrončok - 3.10-2 +- Fix %%python_provide when fed python3.10-foo to obsolete python-foo instead of python--foo + * Tue Jun 01 2021 Miro Hrončok - 3.10-1 - Update main Python to Python 3.10 - https://fedoraproject.org/wiki/Changes/Python3.10 From c487f82ef751f1277a59a80c0fc2cd4efea0b65b Mon Sep 17 00:00:00 2001 From: Ben Burton Date: Mon, 19 Apr 2021 21:02:49 +0200 Subject: [PATCH 142/218] Adapt macros and BRP scripts for %topdir with spaces Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1947416 --- brp-python-bytecompile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index ead7022..71f4d2d 100755 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -38,7 +38,7 @@ function python_bytecompile() local options=$1 local python_binary=$2 local exclude=$3 - local python_libdir=$4 + local python_libdir="$4" local depth=$5 # Not used for Python >= 3.4 local real_libdir=$6 # Not used for Python >= 3.4 @@ -57,7 +57,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - $python_binary -B $options -m compileall -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir + $python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" # # Python 3.4 and higher @@ -73,7 +73,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s $RPM_BUILD_ROOT -p / -e $RPM_BUILD_ROOT $python_libdir + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" else # # Python 3.3 and lower (incl. Python 2) @@ -119,9 +119,9 @@ fi export PYTHONHASHSEED=0 shopt -s nullglob -for python_libdir in `find "$RPM_BUILD_ROOT" -type d|grep -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$"`; +find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir; do - python_binary=$(basename $python_libdir) + python_binary=$(basename "$python_libdir") real_libdir=${python_libdir/$RPM_BUILD_ROOT/} echo "Bytecompiling .py files below $python_libdir using $python_binary" From d905710a8d2fc122ea216c38051ca76fb03b3f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 26 Jun 2021 16:24:21 +0200 Subject: [PATCH 143/218] %pytest: Set $PYTEST_ADDOPTS when %{__pytest_addopts} is defined Related to https://bugzilla.redhat.com/show_bug.cgi?id=1935212 --- macros.python3 | 1 + python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index 480da96..6b19117 100644 --- a/macros.python3 +++ b/macros.python3 @@ -86,4 +86,5 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ %__pytest} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 04fe84c..7f1aa0f 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -31,7 +31,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -109,6 +109,10 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Jun 28 2021 Miro Hrončok - 3.10-3 +- %%pytest: Set $PYTEST_ADDOPTS when %%{__pytest_addopts} is defined +- Related: rhzb#1935212 + * Tue Jun 15 2021 Miro Hrončok - 3.10-2 - Fix %%python_provide when fed python3.10-foo to obsolete python-foo instead of python--foo diff --git a/tests/test_evals.py b/tests/test_evals.py index 43216ca..4a15091 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -211,6 +211,39 @@ def test_pytest_command_suffix(): assert '/usr/bin/pytest-3.6 -v' in lines[-1] +def test_pytest_undefined_addopts_are_not_set(): + lines = rpm_eval('%pytest', __pytest_addopts=None) + assert 'PYTEST_ADDOPTS' not in '\n'.join(lines) + + +def test_pytest_defined_addopts_are_set(): + lines = rpm_eval('%pytest', __pytest_addopts="--ignore=stuff") + assert 'PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} --ignore=stuff"' in '\n'.join(lines) + + +@pytest.mark.parametrize('__pytest_addopts', ['--macronized-option', 'x y z', None]) +def test_pytest_addopts_preserves_envvar(__pytest_addopts): + # this is the line a packager might put in the spec file before running %pytest: + spec_line = 'export PYTEST_ADDOPTS="--exported-option1 --exported-option2"' + + # instead of actually running /usr/bin/pytest, + # we run a small shell script that echoes the tested value for inspection + lines = rpm_eval('%pytest', __pytest_addopts=__pytest_addopts, + __pytest="sh -c 'echo $PYTEST_ADDOPTS'") + + echoed = shell_stdout('\n'.join([spec_line] + lines)) + + # assert all values were echoed + assert '--exported-option1' in echoed + assert '--exported-option2' in echoed + if __pytest_addopts is not None: + assert __pytest_addopts in echoed + + # assert the options are separated + assert 'option--' not in echoed + assert 'z--' not in echoed + + def test_pypi_source_default_name(): urls = rpm_eval('%pypi_source', name='foo', version='6') From bc2b51d6d396e808f4656f50ad4c310483c92f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 30 Jun 2021 14:37:07 +0200 Subject: [PATCH 144/218] Include brp-python-hardlink in python-srpm-macros since it is no longer in RPM 4.17+ See https://src.fedoraproject.org/rpms/redhat-rpm-config/c/def9a339d259ee9767e6dc0cd4b7736cfe0c174f --- brp-python-hardlink | 25 +++++++++++++++++++++++++ macros.python-srpm | 5 +++++ python-rpm-macros.spec | 28 +++++++++++++++++++++++++--- 3 files changed, 55 insertions(+), 3 deletions(-) create mode 100644 brp-python-hardlink diff --git a/brp-python-hardlink b/brp-python-hardlink new file mode 100644 index 0000000..5fd1b43 --- /dev/null +++ b/brp-python-hardlink @@ -0,0 +1,25 @@ +#!/bin/sh + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +hardlink_if_same() { + if cmp -s "$1" "$2" ; then + ln -f "$1" "$2" + return 0 + fi + return 1 +} + +# Hardlink identical *.pyc, *.pyo, and *.opt-[12].pyc. +# Originally from PLD's rpm-build-macros +find "$RPM_BUILD_ROOT" -type f -name "*.pyc" -not -name "*.opt-[12].pyc" | while read pyc ; do + hardlink_if_same "$pyc" "${pyc%c}o" + o1pyc="${pyc%pyc}opt-1.pyc" + hardlink_if_same "$pyc" "$o1pyc" + o2pyc="${pyc%pyc}opt-2.pyc" + hardlink_if_same "$pyc" "$o2pyc" || hardlink_if_same "$o1pyc" "$o2pyc" +done +exit 0 diff --git a/macros.python-srpm b/macros.python-srpm index 844dfeb..5a5799e 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -49,6 +49,11 @@ # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 + +# BRP scripts, they need to be included in redhat-rpm-macros, in %%__os_install_post +%__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink + + # === Macros for Build/Requires tags using Python dist tags === # - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages # - These macros need to be in macros.python-srpm, because BuildRequires tags diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 7f1aa0f..622a7a8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -16,8 +16,16 @@ Source201: python.lua %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py -# macros and lua: MIT, compileall2.py: PSFv2 -License: MIT and Python +# BRP scripts +# This one is from https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-hardlink +# But we don't use a link in case it changes in upstream, there are no "versions" there yet +# This was removed from RPM 4.17+ so we maintain it here instead +Source401: brp-python-hardlink + +# macros and lua: MIT +# compileall2.py: PSFv2 +# brp-python-hardlink: GPLv2+ +License: MIT and Python and GPLv2+ # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. @@ -31,7 +39,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -89,6 +97,16 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ + + +# We define our own BRPs here to use the ones from the %%{buildroot}, +# that way, this package can be built when it includes them for the first time. +# It also ensures that: +# - our BRPs can execute +# - if our BRPs affect this package, we don't need to build it twice +%global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink} + %check # no macros in comments @@ -102,6 +120,7 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -109,6 +128,9 @@ install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Jun 30 2021 Miro Hrončok - 3.10-4 +- Include brp-python-hardlink in python-srpm-macros since it is no longer in RPM 4.17+ + * Mon Jun 28 2021 Miro Hrončok - 3.10-3 - %%pytest: Set $PYTEST_ADDOPTS when %%{__pytest_addopts} is defined - Related: rhzb#1935212 From c2305ea368c963cb8c26250fcce26fb6074b4edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 7 Jul 2021 14:46:04 +0200 Subject: [PATCH 145/218] Introduce %py3_check_import With $PATH and $PYTHONPATH set to the %buildroot, the macro tries to import the given Python 3 module(s). Useful as a smoke test in %check when ruining tests is not feasible. Accepts spaces or commas as separators. Package python-six: %check %py3_check_import six Executing(%check): ... ... + PATH=... + PYTHONPATH=... + PYTHONDONTWRITEBYTECODE=1 + /usr/bin/python3 -c 'import six' + RPM_EC=0 ++ jobs -p + exit 0 %py3_check_import six seven ... + /usr/bin/python3 -c 'import six, seven' Traceback (most recent call last): File "", line 1, in ModuleNotFoundError: No module named 'seven' error: Bad exit status from ... (%check) ... %py3_check_import five, six, seven + /usr/bin/python3 -c 'import five, six, seven' Traceback (most recent call last): File "", line 1, in ModuleNotFoundError: No module named 'five' error: Bad exit status from ... (%check) Package python-packaging: %py3_check_import packaging, packaging.markers packaging.requirements, packaging.tags Executing(%check): ... ... + PATH=... + PYTHONPATH=... + PYTHONDONTWRITEBYTECODE=1 + /usr/bin/python3 -c 'import packaging, packaging.markers, packaging.requirements, packaging.tags' + RPM_EC=0 ++ jobs -p + exit 0 %py3_check_import packaging, packaging.markers packaging.notachance, packaging.tags ... + /usr/bin/python3 -c 'import packaging, packaging.markers, packaging.notachance, packaging.tags' Traceback (most recent call last): File "", line 1, in ModuleNotFoundError: No module named 'packaging.notachance' error: Bad exit status from ... (%check) --- macros.python | 13 ++++++++++++ macros.python3 | 13 ++++++++++++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 45 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index f0c3f14..04c47c5 100644 --- a/macros.python +++ b/macros.python @@ -66,6 +66,19 @@ done } +# With $PATH and $PYTHONPATH set to the %%buildroot, +# try to import the given Python module(s). +# Useful as a smoke test in %%check when running tests is not feasible. +# Use spaces or commas as separators. +%py_check_import() %{expand:\\\ + (cd %{_topdir} &&\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" + ) +} + %python_provide() %{lua: local python = require "fedora.srpm.python" function string.starts(String,Start) diff --git a/macros.python3 b/macros.python3 index 6b19117..93b4199 100644 --- a/macros.python3 +++ b/macros.python3 @@ -64,6 +64,19 @@ done } +# With $PATH and $PYTHONPATH set to the %%buildroot, +# try to import the given Python 3 module(s). +# Useful as a smoke test in %%check when running tests is not feasible. +# Use spaces or commas as separators. +%py3_check_import() %{expand:\\\ + (cd %{_topdir} &&\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" + ) +} + # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. %pycached() %{lua: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 622a7a8..46e2d1b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -39,7 +39,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -128,6 +128,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Jul 07 2021 Miro Hrončok - 3.10-5 +- Introduce %%py3_check_import + * Wed Jun 30 2021 Miro Hrončok - 3.10-4 - Include brp-python-hardlink in python-srpm-macros since it is no longer in RPM 4.17+ diff --git a/tests/test_evals.py b/tests/test_evals.py index 4a15091..631f234 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,6 +1,7 @@ import os import subprocess import platform +import re import sys import textwrap @@ -548,6 +549,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%py_install', '%py_install_egg', '%py_install_wheel', + '%py_check_import', ]) @@ -600,3 +602,46 @@ def test_python3_sitearch_value(lib): macro = '%python3_sitearch' assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] assert rpm_eval(macro) == [f'/usr/{lib}/python{X_Y}/site-packages'] + + +@pytest.mark.parametrize( + 'args, imports', + [ + ('six', 'six'), + ('five six seven', 'five, six, seven'), + ('six,seven, eight', 'six, seven, eight'), + ('six.quarter six.half,, SIX', 'six.quarter, six.half, SIX'), + ] +) +@pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/python3.6']) +def test_py3_check_import(args, imports, __python3, lib): + x_y = X_Y + macors = { + 'buildroot': 'BUILDROOT', + '_topdir': 'TOPDIR', + } + if __python3 is not None: + macors['__python3'] = __python3 + # If the __python3 command has version at the end, parse it and expect it. + # Note that the command is used to determine %python3_sitelib and %python3_sitearch, + # so we only test known CPython schemes here and not PyPy for simplicity. + # We also only test main Python + 3.6 because those are required by the CI config. + if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): + x_y = match.group(1) + + lines = rpm_eval(f'%py3_check_import {args}', **macors) + + # An equality check is a bit inflexible here, + # every time we change the macro we need to change this test. + # However actually executing it and verifying the result is much harder :/ + # At least, let's make the lines saner to check: + lines = [line.rstrip('\\').strip() for line in lines] + expected = textwrap.dedent(fr""" + (cd TOPDIR && + PATH="BUILDROOT/usr/bin:$PATH" + PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" + PYTHONDONTWRITEBYTECODE=1 + {__python3 or '/usr/bin/python3'} -c "import {imports}" + ) + """) + assert lines == expected.splitlines() From b1488aa40cfe04e9e8347ac902082a3cc8f17fc2 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 23 Jul 2021 09:18:36 +0000 Subject: [PATCH 146/218] - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a007695..937ea48 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -141,6 +141,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild + * Wed Jul 07 2021 Miro Hrončok - 3.10-6 - Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros From 76209d7bf3cb104ea8edba3e8b7941fd6d3b53a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 16 Aug 2021 17:03:38 +0200 Subject: [PATCH 147/218] Fedora CI eval tests: Make the python3.6 dependency optional It is not available on RHEL 9, where we would like to be able to run the tests. See https://bugzilla.redhat.com/show_bug.cgi?id=1984407 --- tests/test_evals.py | 88 +++++++++++++++++++++++++++++++++++---------- tests/tests.yml | 2 +- 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/tests/test_evals.py b/tests/test_evals.py index 631f234..9061e39 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -51,6 +51,31 @@ def lib(): return lib_eval +def get_alt_x_y(): + """ + Some tests require alternate Python version to be installed. + In order to allow any Python version (or none at all), + this function/fixture exists. + You can control the behavior by setting the $ALTERNATE_PYTHON_VERSION + environment variable to X.Y (e.g. 3.6) or SKIP. + The environment variable must be set. + """ + env_name = "ALTERNATE_PYTHON_VERSION" + alternate_python_version = os.getenv(env_name, "") + if alternate_python_version.upper() == "SKIP": + pytest.skip(f"${env_name} set to SKIP") + if not alternate_python_version: + raise ValueError(f"${env_name} must be set, " + f"set it to SKIP if you want to skip tests that " + f"require alternate Python version.") + if not re.match(r"^\d+\.\d+$", alternate_python_version): + raise ValueError(f"${env_name} must be X.Y") + return alternate_python_version + +# We don't use the decorator, to be able to call the function itselef +alt_x_y = pytest.fixture(scope="session")(get_alt_x_y) + + def shell_stdout(script): return subprocess.check_output(script, env={**os.environ, 'LANG': 'C.utf-8'}, @@ -82,8 +107,8 @@ def test_py3_dist(): assert rpm_eval(f'%py3_dist Aha[Boom] a') == ['python3dist(aha[boom]) python3dist(a)'] -def test_py3_dist_with_python3_pkgversion_redefined(): - assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion="3.6") == ['python3.6dist(aha[boom]) python3.6dist(a)'] +def test_py3_dist_with_python3_pkgversion_redefined(alt_x_y): + assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion=alt_x_y) == [f'python{alt_x_y}dist(aha[boom]) python{alt_x_y}dist(a)'] def test_python_provide_python(): @@ -208,8 +233,12 @@ def test_pytest_different_command(): def test_pytest_command_suffix(): lines = rpm_eval('%pytest -v') assert '/usr/bin/pytest -v' in lines[-1] - lines = rpm_eval('%pytest -v', python3_pkgversion="3.6", python3_version="3.6") - assert '/usr/bin/pytest-3.6 -v' in lines[-1] + +# this test does not require alternate Pythons to be installed +@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12']) +def test_pytest_command_suffix_alternate_pkgversion(version): + lines = rpm_eval('%pytest -v', python3_pkgversion=version, python3_version=version) + assert f'/usr/bin/pytest-{version} -v' in lines[-1] def test_pytest_undefined_addopts_are_not_set(): @@ -344,11 +373,14 @@ def test_pycached_in_sitearch(lib): ] -def test_pycached_in_36(): - lines = rpm_eval('%pycached /usr/lib/python3.6/site-packages/foo*.py') +# this test does not require alternate Pythons to be installed +@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12']) +def test_pycached_with_alternate_version(version): + version_nodot = version.replace('.', '') + lines = rpm_eval(f'%pycached /usr/lib/python{version}/site-packages/foo*.py') assert lines == [ - '/usr/lib/python3.6/site-packages/foo*.py', - '/usr/lib/python3.6/site-packages/__pycache__/foo*.cpython-36{,.opt-?}.pyc' + f'/usr/lib/python{version}/site-packages/foo*.py', + f'/usr/lib/python{version}/site-packages/__pycache__/foo*.cpython-{version_nodot}{{,.opt-?}}.pyc' ] @@ -580,30 +612,46 @@ def test_ext_suffix(): assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"] -def test_python_sitelib_value(): +def test_python_sitelib_value_python3(): macro = '%python_sitelib' - assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages'] -def test_python3_sitelib_value(): +def test_python_sitelib_value_alternate_python(alt_x_y): + macro = '%python_sitelib' + assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/lib/python{alt_x_y}/site-packages'] + + +def test_python3_sitelib_value_default(): macro = '%python3_sitelib' - assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] assert rpm_eval(macro) == [f'/usr/lib/python{X_Y}/site-packages'] -def test_python_sitearch_value(lib): +def test_python3_sitelib_value_alternate_python(alt_x_y): + macro = '%python3_sitelib' + assert rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == [f'/usr/lib/python{alt_x_y}/site-packages'] + + +def test_python_sitearch_value_python3(lib): macro = '%python_sitearch' - assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] assert rpm_eval(macro, __python='%__python3') == [f'/usr/{lib}/python{X_Y}/site-packages'] -def test_python3_sitearch_value(lib): +def test_python_sitearch_value_alternate_python(lib, alt_x_y): + macro = '%python_sitearch' + assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/{lib}/python{alt_x_y}/site-packages'] + + +def test_python3_sitearch_value_default(lib): macro = '%python3_sitearch' - assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] assert rpm_eval(macro) == [f'/usr/{lib}/python{X_Y}/site-packages'] +def test_python3_sitearch_value_alternate_python(lib, alt_x_y): + macro = '%python3_sitearch' + assert rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == [f'/usr/{lib}/python{alt_x_y}/site-packages'] + + @pytest.mark.parametrize( 'args, imports', [ @@ -613,7 +661,10 @@ def test_python3_sitearch_value(lib): ('six.quarter six.half,, SIX', 'six.quarter, six.half, SIX'), ] ) -@pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/python3.6']) +@pytest.mark.parametrize('__python3', + [None, + f'/usr/bin/python{X_Y}', + '/usr/bin/pythonX.Y']) def test_py3_check_import(args, imports, __python3, lib): x_y = X_Y macors = { @@ -621,11 +672,12 @@ def test_py3_check_import(args, imports, __python3, lib): '_topdir': 'TOPDIR', } if __python3 is not None: + if 'X.Y' in __python3: + __python3 = __python3.replace('X.Y', get_alt_x_y()) macors['__python3'] = __python3 # If the __python3 command has version at the end, parse it and expect it. # Note that the command is used to determine %python3_sitelib and %python3_sitearch, # so we only test known CPython schemes here and not PyPy for simplicity. - # We also only test main Python + 3.6 because those are required by the CI config. if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) diff --git a/tests/tests.yml b/tests/tests.yml index d6090ec..117b999 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -15,7 +15,7 @@ tests: - pytest: dir: . - run: pytest -v + run: ALTERNATE_PYTHON_VERSION=3.6 pytest -v - manual_byte_compilation: dir: . run: rpmbuild -ba pythontest.spec From 37bf640f37e274e0b75365213caa4725583ab14a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 9 Sep 2021 16:54:04 +0200 Subject: [PATCH 148/218] Use --hardlink-dupes in %py_byte_compile and brp-python-bytecompile (for Python 3.9+) Resolves: rhbz#1977895 --- brp-python-bytecompile | 10 ++++++++-- macros.pybytecompile | 2 +- python-rpm-macros.spec | 7 ++++++- tests/pythontest.spec | 24 ++++++++++++++++++++++-- 4 files changed, 37 insertions(+), 6 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 71f4d2d..cbc5830 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -29,7 +29,6 @@ fi # Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks # When we drop support for Python 2, we'd be able to use all compileall2 features like: # - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir -# - -o 0 -o 1 to produce multiple files in one run - each with a different optimization level - instead of $options # - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() # These changes will make this script much simpler # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. @@ -49,6 +48,12 @@ function python_bytecompile() # if [ "$python_version" -ge 39 ]; then + # For Python 3.9+, we compile all opt levels in one go: only + # when $options is empty. + if [ -n "$options" ]; then + return + fi + [ ! -z $exclude ] && exclude="-x '$exclude'" # -q disables verbose output # -f forces the process to overwrite existing compiled files @@ -57,7 +62,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - $python_binary -B $options -m compileall -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" + $python_binary -B -m compileall -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" # # Python 3.4 and higher @@ -133,6 +138,7 @@ do fi # Generate optimized (.pyo) byte-compiled files. + # N.B. For Python 3.9+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error diff --git a/macros.pybytecompile b/macros.pybytecompile index b58fff4..6c89c8b 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -34,7 +34,7 @@ py3_byte_compile () {\ py39_byte_compile () {\ python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ + $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 937ea48..e39b07f 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -141,6 +141,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu Sep 09 2021 Petr Viktorin - 3.10-8 +- Use --hardlink-dupes in %%py_byte_compile and brp-python-bytecompile + (for Python 3.9+) +- Resolves: rhbz#1977895 + * Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild diff --git a/tests/pythontest.spec b/tests/pythontest.spec index a0f052c..182ba59 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -19,15 +19,35 @@ echo "print()" > %{buildroot}%{basedir}/directory/to/test/recursion/file_in_dir. %py_byte_compile %{python3} %{buildroot}%{basedir}/file.py %py_byte_compile %{python3} %{buildroot}%{basedir}/directory +# Files in sitelib are compiled automatically by brp-python-bytecompile +mkdir -p %{buildroot}%{python3_sitelib}/directory/ +echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py + %check +LOCATIONS="%{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/" + # Count .py and .pyc files -PY=$(find %{buildroot}%{basedir} -name "*.py" | wc -l) -PYC=$(find %{buildroot}%{basedir} -name "*.pyc" | wc -l) +PY=$(find $LOCATIONS -name "*.py" | wc -l) +PYC=$(find $LOCATIONS -name "*.pyc" | wc -l) + +# We should have 3 .py files +test $PY -eq 3 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files test $(expr $PY \* 2) -eq $PYC +# In this case the .pyc files should be identical across omtimization levels +# (they don't use docstrings and assert staements) +# So they should be hardlinked; the number of distinct inodes should match the +# number of source files. (Or be smaller, if the dupe detection is done +# across all files.) + +INODES=$(stat --format %i $(find $LOCATIONS -name "*.pyc") | sort -u | wc -l) +test $PY -ge $INODES + + %files %pycached %{basedir}/file.py %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py +%pycached %{python3_sitelib}/directory/file.py From dd8caa5aa39eda2a1d8fb5ff28fd7a6094ba1fd5 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Sep 2021 15:34:10 +0200 Subject: [PATCH 149/218] Use --hardlink-dupes for Python 3.4+ as well --- brp-python-bytecompile | 68 ++++++++++++++++++++---------------------- macros.pybytecompile | 2 +- python-rpm-macros.spec | 2 +- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index cbc5830..7e20608 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -26,12 +26,13 @@ fi # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. # Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks + # When we drop support for Python 2, we'd be able to use all compileall2 features like: # - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir # - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() # These changes will make this script much simpler -# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. function python_bytecompile() { local options=$1 @@ -41,45 +42,40 @@ function python_bytecompile() local depth=$5 # Not used for Python >= 3.4 local real_libdir=$6 # Not used for Python >= 3.4 - python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") - # - # Python 3.9 and higher - # - if [ "$python_version" -ge 39 ]; then + # + # Python 3.4 and higher + # + if [ "$python_version" -ge 34 ]; then - # For Python 3.9+, we compile all opt levels in one go: only - # when $options is empty. - if [ -n "$options" ]; then - return - fi + # We compile all opt levels in one go: only when $options is empty. + if [ -n "$options" ]; then + return + fi - [ ! -z $exclude ] && exclude="-x '$exclude'" - # -q disables verbose output - # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex - # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below - # -s strips $RPM_BUILD_ROOT from the path - # -p prepends the leading slash to the path to make it absolute - $python_binary -B -m compileall -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" + if [ "$python_version" -ge 39 ]; then + # For Pyhon 3.9+, use the standard library + compileall_module=compileall + else + # For older Pythons, use compileall2 + compileall_module=compileall2 + fi - # - # Python 3.4 and higher - # - elif [ "$python_version" -ge 34 ]; then + [ ! -z $exclude ] && exclude="-x '$exclude'" - [ ! -z $exclude ] && exclude="-x '$exclude'" - # /usr/lib/rpm/redhat/ contains compileall2 Python module - # -q disables verbose output - # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex - # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below - # -s strips $RPM_BUILD_ROOT from the path - # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B $options -m compileall2 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / -e "$RPM_BUILD_ROOT" "$python_libdir" - else + # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib + # -o 0 -o 1 are the optimization levels + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex + # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" + + else # # Python 3.3 and lower (incl. Python 2) # @@ -138,7 +134,7 @@ do fi # Generate optimized (.pyo) byte-compiled files. - # N.B. For Python 3.9+, this call does nothing + # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error diff --git a/macros.pybytecompile b/macros.pybytecompile index 6c89c8b..9dabee0 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -28,7 +28,7 @@ py2_byte_compile () {\ py3_byte_compile () {\ python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e39b07f..624a66e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -143,7 +143,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog * Thu Sep 09 2021 Petr Viktorin - 3.10-8 - Use --hardlink-dupes in %%py_byte_compile and brp-python-bytecompile - (for Python 3.9+) + (for Python 3) - Resolves: rhbz#1977895 * Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 From 0044db1e8af178a9c9f5a6bcf653611816dbc3b3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Sep 2021 15:36:57 +0200 Subject: [PATCH 150/218] Remove unneeded arguments from the python_bytecompile function - For depth, use sys.getrecursionlimit() - the default from bytecompile2 - we'll get a RecursionError before it's exceeded, anyway - Set real_libdir locally --- brp-python-bytecompile | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 7e20608..ce73856 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -16,31 +16,16 @@ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi -# Figure out how deep we need to descend. We could pick an insanely high -# number and hope it's enough, but somewhere, somebody's sure to run into it. -depth=`(find "$RPM_BUILD_ROOT" -type f -name "*.py" -print0 ; echo /) | \ - xargs -0 -n 1 dirname | sed 's,[^/],,g' | sort -u | tail -n 1 | wc -c` -if [ -z "$depth" -o "$depth" -le "1" ]; then - exit 0 -fi - # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. # Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks - -# When we drop support for Python 2, we'd be able to use all compileall2 features like: -# - -s and -p options to manipulate with a path baked into pyc files instead of $real_libdir -# - removed useless $depth - both compileall and compileall2 are limited by sys.getrecursionlimit() -# These changes will make this script much simpler function python_bytecompile() { local options=$1 local python_binary=$2 local exclude=$3 local python_libdir="$4" - local depth=$5 # Not used for Python >= 3.4 - local real_libdir=$6 # Not used for Python >= 3.4 python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") @@ -80,11 +65,13 @@ function python_bytecompile() # Python 3.3 and lower (incl. Python 2) # +local real_libdir=${python_libdir/$RPM_BUILD_ROOT/} + cat << EOF | $python_binary $options import compileall, sys, os, re python_libdir = "$python_libdir" -depth = $depth +depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" exclude = r"$exclude" @@ -123,11 +110,10 @@ shopt -s nullglob find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir; do python_binary=$(basename "$python_libdir") - real_libdir=${python_libdir/$RPM_BUILD_ROOT/} echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. - python_bytecompile "" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + python_bytecompile "" "$python_binary" "" "$python_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 @@ -135,7 +121,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing - python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$depth" "$real_libdir" + python_bytecompile "-O" "$python_binary" "" "$python_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 From 77cc1a43a293efea384ade979f7682dc69782cc3 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 10 Sep 2021 16:11:24 +0200 Subject: [PATCH 151/218] Test bytecompilation & hardlinking with 3.8 and 2.7 --- tests/pythontest.spec | 30 +++++++++++++++++++++++++----- tests/tests.yml | 1 + 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 182ba59..3b707c5 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,11 +1,18 @@ %global basedir /opt/test/byte_compilation +# We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 +# Test with a representative of each. +%global python36_sitelib /usr/lib/python3.6/site-packages +%global python27_sitelib /usr/lib/python2.7/site-packages + Name: pythontest Version: 0 Release: 0 Summary: ... License: MIT BuildRequires: python3-devel +BuildRequires: python3.6 +BuildRequires: python2.7 %description ... @@ -23,15 +30,26 @@ echo "print()" > %{buildroot}%{basedir}/directory/to/test/recursion/file_in_dir. mkdir -p %{buildroot}%{python3_sitelib}/directory/ echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py +mkdir -p %{buildroot}%{python36_sitelib}/directory/ +echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py + +mkdir -p %{buildroot}%{python27_sitelib}/directory/ +echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py + %check -LOCATIONS="%{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/" +LOCATIONS=" + %{buildroot}%{basedir} + %{buildroot}%{python3_sitelib}/directory/ + %{buildroot}%{python36_sitelib}/directory/ + %{buildroot}%{python27_sitelib}/directory/ +" # Count .py and .pyc files PY=$(find $LOCATIONS -name "*.py" | wc -l) -PYC=$(find $LOCATIONS -name "*.pyc" | wc -l) +PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) -# We should have 3 .py files -test $PY -eq 3 +# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7) +test $PY -eq 5 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files @@ -43,7 +61,7 @@ test $(expr $PY \* 2) -eq $PYC # number of source files. (Or be smaller, if the dupe detection is done # across all files.) -INODES=$(stat --format %i $(find $LOCATIONS -name "*.pyc") | sort -u | wc -l) +INODES=$(stat --format %i $(find $LOCATIONS -name "*.py[co]") | sort -u | wc -l) test $PY -ge $INODES @@ -51,3 +69,5 @@ test $PY -ge $INODES %pycached %{basedir}/file.py %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py %pycached %{python3_sitelib}/directory/file.py +%pycached %{python36_sitelib}/directory/file.py +%{python27_sitelib}/directory/file.py* diff --git a/tests/tests.yml b/tests/tests.yml index 117b999..1c0478a 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -26,4 +26,5 @@ - python3-devel - python3-pytest - python3.6 + - python2.7 From 5b578a851f09c93b8e105ab22511f004f6e5c9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 3 Sep 2021 16:06:03 +0200 Subject: [PATCH 152/218] Set $RPM_BUILD_ROOT in %{python3_...} macros, for alternate sysconfig install scheme Our Pythons currently patches distutils to install packages to /usr/lib(64)/pythonX.Y/site-packages when the $RPM_BUILD_ROOT environment variable is set (and to /usr/local/lib(64)/pythonX.Y/site-packages otherwise). With the deprecation of distutils [1] we want to change the patch to create and use a different sysconfig install scheme [2]. However, we have realized that macros defined as %(%{__python3} ...) don't "see" the environment variables set by rpmbuild because they are expanded earlier and hence e.g. %{python3_sitelib} evaluates to /usr/local/lib/python3.X/site-packages -- which is not desired. To be able to reliably detect an RPM build environment by checking the presence of the $RPM_BUILD_ROOT environment variable, we manually set it in the macro definitions. Since %{buildroot} in not fully populated (e.g. it can expand literally to /home/anna/rpmbuild/BUILDROOT/%{NAME}-%{VERSION}-%{RELEASE}.x86_64), we don't use it here. The variable simply needs to present in the environment. See also the analysis of the build failures when this is not done [3]. [1] https://www.python.org/dev/peps/pep-0632/ [2] https://bugs.python.org/issue43976 [3] https://src.fedoraproject.org/rpms/python3.10/pull-request/63#comment-79042 --- macros.python | 16 +++++++++------- macros.python3 | 16 +++++++++------- python-rpm-macros.spec | 6 +++++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/macros.python b/macros.python index 04c47c5..39b4120 100644 --- a/macros.python +++ b/macros.python @@ -1,12 +1,14 @@ # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros -%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") -%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") -%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") +%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") +%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index 93b4199..7db969e 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,10 +1,12 @@ -%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") -%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") -%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") +%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") +%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 624a66e..b0364e2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -141,6 +141,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu Sep 09 2021 Miro Hrončok - 3.10-9 +- Set $RPM_BUILD_ROOT in %%{python3_...} macros + to allow selecting alternate sysconfig install scheme based on that variable + * Thu Sep 09 2021 Petr Viktorin - 3.10-8 - Use --hardlink-dupes in %%py_byte_compile and brp-python-bytecompile (for Python 3) From 7b546cae36d194b01389ed900a9c4b1ddcbb1f8e Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 12 Oct 2021 15:50:50 +0200 Subject: [PATCH 153/218] Non-existing path in py_reproducible_pyc_path causes build to fail --- brp-fix-pyc-reproducibility | 2 ++ python-rpm-macros.spec | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 4118d97..536a126 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -15,4 +15,6 @@ if [ ! -x /usr/bin/marshalparser ]; then exit 1 fi +# Set pipefail so if $path_to_fix does not exist, the build fails +set -o pipefail find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b0364e2..13d87d1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 10%{?dist} BuildArch: noarch @@ -141,6 +141,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Oct 12 2021 Lumír Balhar - 3.10-10 +- Non-existing path in py_reproducible_pyc_path causes build to fail +Resolves: rhbz#2011056 + * Thu Sep 09 2021 Miro Hrončok - 3.10-9 - Set $RPM_BUILD_ROOT in %%{python3_...} macros to allow selecting alternate sysconfig install scheme based on that variable From 9b797df44dc6c184c5181f10bee24461287f84f3 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 29 Sep 2021 12:47:19 +0200 Subject: [PATCH 154/218] Define a new macros %python_wheel_dir and %python_wheel_pkg_prefix --- macros.python-srpm | 14 ++++++++++++++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index f115a85..dcc7023 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -49,6 +49,20 @@ # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 +# Define where Python wheels will be stored and the prefix of -wheel packages +# - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that +# install packages into `/usr/share/python-wheels`. Both names are not +# versioned, because they're used by all Python 3 stacks. +# - In RHEL we want wheel packages named e.g. `python3-pip-wheel` and +# `python3.11-pip-wheel` that install packages into similarly versioned +# locations. We want each Python stack in RHEL to have their own wheels, +# because the main python3 wheels (which we can't upgrade) will likely be +# quite old by the time we're adding new alternate Python stacks. +# - In ELN we want to follow Fedora, because builds for ELN and Fedora rawhide +# need to be interoperable. +%python_wheel_pkg_prefix python%{?rhel:%{!?eln:%{python3_pkgversion}}} +%python_wheel_dir %{_datadir}/%{python_wheel_pkg_prefix}-wheels + ### BRP scripts (and related macros) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 13d87d1..0b73c3b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 10%{?dist} +Release: 11%{?dist} BuildArch: noarch @@ -141,6 +141,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Oct 20 2021 Tomas Orsava - 3.10-11 +- Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix + * Tue Oct 12 2021 Lumír Balhar - 3.10-10 - Non-existing path in py_reproducible_pyc_path causes build to fail Resolves: rhbz#2011056 diff --git a/tests/test_evals.py b/tests/test_evals.py index 9061e39..f86284d 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -220,6 +220,24 @@ def test_py_provides_with_evr(): assert len(lines) == 3 +def test_python_wheel_pkg_prefix(): + assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None) == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln='1') == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None) == ['python3'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['python3.10'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['python3.11'] + + +def test_python_wheel_dir(): + assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None) == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln='1') == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None) == ['/usr/share/python3-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['/usr/share/python3.10-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['/usr/share/python3.11-wheels'] + + def test_pytest_passes_options_naturally(): lines = rpm_eval('%pytest -k foo') assert '/usr/bin/pytest -k foo' in lines[-1] From 2d0673afb17814e4bd6b7a281574b1d0e1fdb3e3 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 18 Oct 2021 16:33:04 +0200 Subject: [PATCH 155/218] Add new options for %%py{3}_check_import: -f, -t, -e -f: optionally read a file with module names to test -t: bool flag - if set, filter only top-level modules -e: optionally exclude module names matching the given glob (Unix shell-style wildcards) Importing all modules may cause bogus failures in some cases, eg. when the imported code assumes there is an existing graphical window. Such behaviour may be by design, hence for automatic processing it's more convinient to - in some cases - check only for top-level modules or filter out the troublemakers. --- import_all_modules.py | 152 ++++++++++++ macros.python | 13 +- macros.python3 | 13 +- python-rpm-macros.spec | 11 +- tests/test_evals.py | 25 +- tests/test_import_all_modules.py | 392 +++++++++++++++++++++++++++++++ tests/tests.yml | 2 +- 7 files changed, 581 insertions(+), 27 deletions(-) create mode 100644 import_all_modules.py create mode 100644 tests/test_import_all_modules.py diff --git a/import_all_modules.py b/import_all_modules.py new file mode 100644 index 0000000..7536133 --- /dev/null +++ b/import_all_modules.py @@ -0,0 +1,152 @@ +'''Script to perform import of each module given to %%py_check_import +''' +import argparse +import importlib +import fnmatch +import re +import sys + +from contextlib import contextmanager +from pathlib import Path + + +def read_modules_files(file_paths): + '''Read module names from the files (modules must be newline separated). + + Return the module names list or, if no files were provided, an empty list. + ''' + + if not file_paths: + return [] + + modules = [] + for file in file_paths: + file_contents = file.read_text() + modules.extend(file_contents.split()) + return modules + + +def read_modules_from_cli(argv): + '''Read module names from command-line arguments (space or comma separated). + + Return the module names list. + ''' + + if not argv: + return [] + + # %%py3_check_import allows to separate module list with comma or whitespace, + # we need to unify the output to a list of particular elements + modules_as_str = ' '.join(argv) + modules = re.split(r'[\s,]+', modules_as_str) + return modules + + +def filter_top_level_modules_only(modules): + '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'. + + Return the list of top-level modules. + ''' + + return [module for module in modules if '.' not in module] + + +def any_match(text, globs): + '''Return True if any of given globs fnmatchcase's the given text.''' + + return any(fnmatch.fnmatchcase(text, g) for g in globs) + + +def exclude_unwanted_module_globs(globs, modules): + '''Filter out entries which match the either of the globs given as argv. + + Return the list of filtered modules. + ''' + + return [m for m in modules if not any_match(m, globs)] + + +def read_modules_from_all_args(args): + '''Return a joined list of modules from all given command-line arguments. + ''' + + modules = read_modules_files(args.filename) + modules.extend(read_modules_from_cli(args.modules)) + if args.exclude: + modules = exclude_unwanted_module_globs(args.exclude, modules) + + if args.top_level: + modules = filter_top_level_modules_only(modules) + + # Error when someone accidentally managed to filter out everything + if len(modules) == 0: + raise ValueError('No modules to check were left') + + return modules + + +def import_modules(modules): + '''Procedure to perform import check for each module name from the given list of modules. + ''' + + for module in modules: + print('Check import:', module, file=sys.stderr) + importlib.import_module(module) + + +def argparser(): + parser = argparse.ArgumentParser( + description='Generate list of all importable modules for import check.' + ) + parser.add_argument( + 'modules', nargs='*', + help=('Add modules to check the import (space or comma separated).'), + ) + parser.add_argument( + '-f', '--filename', action='append', type=Path, + help='Add importable module names list from file.', + ) + parser.add_argument( + '-t', '--top-level', action='store_true', + help='Check only top-level modules.', + ) + parser.add_argument( + '-e', '--exclude', action='append', + help='Provide modules globs to be excluded from the check.', + ) + return parser + + +@contextmanager +def remove_unwanteds_from_sys_path(): + '''Remove cwd and this script's parent from sys.path for the import test. + Bring the original contents back after import is done (or failed) + ''' + + cwd_absolute = Path.cwd().absolute() + this_file_parent = Path(__file__).parent.absolute() + old_sys_path = list(sys.path) + for path in old_sys_path: + if Path(path).absolute() in (cwd_absolute, this_file_parent): + sys.path.remove(path) + try: + yield + finally: + sys.path = old_sys_path + + +def main(argv=None): + + cli_args = argparser().parse_args(argv) + + if not cli_args.modules and not cli_args.filename: + raise ValueError('No modules to check were provided') + + modules = read_modules_from_all_args(cli_args) + + with remove_unwanteds_from_sys_path(): + import_modules(modules) + + +if __name__ == '__main__': + main() diff --git a/macros.python b/macros.python index 39b4120..6c0ad5b 100644 --- a/macros.python +++ b/macros.python @@ -69,16 +69,17 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python module(s). +# try to import the Python module(s) given as command-line args or read from file (-f). +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} } %python_provide() %{lua: diff --git a/macros.python3 b/macros.python3 index 7db969e..d9a4c06 100644 --- a/macros.python3 +++ b/macros.python3 @@ -67,16 +67,17 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python 3 module(s). +# try to import the Python 3 module(s) given as command-line args or read from file (-f). +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py3_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py3_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} } # This only supports Python 3.5+ and will never work with Python 2. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0b73c3b..373e389 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -15,6 +15,7 @@ Source201: python.lua # Python code %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py +Source302: import_all_modules.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -31,6 +32,7 @@ Source402: brp-python-hardlink Source403: brp-fix-pyc-reproducibility # macros and lua: MIT +# import_all_modules.py: MIT # compileall2.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -47,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 11%{?dist} +Release: 12%{?dist} BuildArch: noarch @@ -105,6 +107,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -131,6 +134,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -141,6 +145,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Oct 25 2021 Karolina Surma - 3.10-12 +- Introduce -f (read from file) option to %%py{3}_check_import +- Introduce -t (filter top-level modules) option to %%py{3}_check_import +- Introduce -e (exclude module globs) option to %%py{3}_check_import + * Wed Oct 20 2021 Tomas Orsava - 3.10-11 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix diff --git a/tests/test_evals.py b/tests/test_evals.py index f86284d..d136cc1 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -671,35 +671,36 @@ def test_python3_sitearch_value_alternate_python(lib, alt_x_y): @pytest.mark.parametrize( - 'args, imports', + 'args', [ - ('six', 'six'), - ('five six seven', 'five, six, seven'), - ('six,seven, eight', 'six, seven, eight'), - ('six.quarter six.half,, SIX', 'six.quarter, six.half, SIX'), + 'six', + '-f foo.txt', + '-t -f foo.txt six, seven', + '-e "foo*" -f foo.txt six, seven', + 'six.quarter six.half,, SIX', ] ) @pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/pythonX.Y']) -def test_py3_check_import(args, imports, __python3, lib): +def test_py3_check_import(args, __python3, lib): x_y = X_Y - macors = { + macros = { 'buildroot': 'BUILDROOT', - '_topdir': 'TOPDIR', + '_rpmconfigdir': 'RPMCONFIGDIR', } if __python3 is not None: if 'X.Y' in __python3: __python3 = __python3.replace('X.Y', get_alt_x_y()) - macors['__python3'] = __python3 + macros['__python3'] = __python3 # If the __python3 command has version at the end, parse it and expect it. # Note that the command is used to determine %python3_sitelib and %python3_sitearch, # so we only test known CPython schemes here and not PyPy for simplicity. if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) - lines = rpm_eval(f'%py3_check_import {args}', **macors) + lines = rpm_eval(f'%py3_check_import {args}', **macros) # An equality check is a bit inflexible here, # every time we change the macro we need to change this test. @@ -707,11 +708,9 @@ def test_py3_check_import(args, imports, __python3, lib): # At least, let's make the lines saner to check: lines = [line.rstrip('\\').strip() for line in lines] expected = textwrap.dedent(fr""" - (cd TOPDIR && PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -c "import {imports}" - ) + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {args} """) assert lines == expected.splitlines() diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py new file mode 100644 index 0000000..490ef1d --- /dev/null +++ b/tests/test_import_all_modules.py @@ -0,0 +1,392 @@ +from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import main as modules_main +from import_all_modules import read_modules_from_cli, filter_top_level_modules_only + +from pathlib import Path + +import pytest +import shlex +import sys + + +@pytest.fixture(autouse=True) +def preserve_sys_path(): + original_sys_path = list(sys.path) + yield + sys.path = original_sys_path + + +@pytest.fixture(autouse=True) +def preserve_sys_modules(): + original_sys_modules = dict(sys.modules) + yield + sys.modules = original_sys_modules + + +@pytest.mark.parametrize( + 'args, imports', + [ + ('six', ['six']), + ('five six seven', ['five', 'six', 'seven']), + ('six,seven, eight', ['six', 'seven', 'eight']), + ('six.quarter six.half,, SIX', ['six.quarter', 'six.half', 'SIX']), + ] +) +def test_read_modules_from_cli(args, imports): + argv = shlex.split(args) + cli_args = argparser().parse_args(argv) + assert read_modules_from_cli(cli_args.modules) == imports + + +@pytest.mark.parametrize( + 'all_mods, imports', + [ + (['six'], ['six']), + (['five', 'six', 'seven'], ['five', 'six', 'seven']), + (['six.seven', 'eight'], ['eight']), + (['SIX', 'six.quarter', 'six.half.and.sth', 'seven'], ['SIX', 'seven']), + ], +) +def test_filter_top_level_modules_only(all_mods, imports): + assert filter_top_level_modules_only(all_mods) == imports + + +@pytest.mark.parametrize( + 'globs, expected', + [ + (['*.*'], ['foo', 'boo']), + (['?oo'], ['foo.bar', 'foo.bar.baz', 'foo.baz']), + (['*.baz'], ['foo', 'foo.bar', 'boo']), + (['foo'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']), + (['foo*'], ['boo']), + (['foo*', '*bar'], ['boo']), + (['foo', 'bar'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']), + (['*'], []), + ] +) +def test_exclude_unwanted_module_globs(globs, expected): + my_modules = ['foo', 'foo.bar', 'foo.bar.baz', 'foo.baz', 'boo'] + tested = exclude_unwanted_module_globs(globs, my_modules) + assert tested == expected + + +def test_cli_with_all_args(): + '''A smoke test, all args must be parsed correctly.''' + mods = ['foo', 'foo.bar', 'baz'] + files = ['-f', './foo'] + top = ['-t'] + exclude = ['-e', 'foo*'] + cli_args = argparser().parse_args([*mods, *files, *top, *exclude]) + + assert cli_args.filename == [Path('foo')] + assert cli_args.top_level is True + assert cli_args.modules == ['foo', 'foo.bar', 'baz'] + assert cli_args.exclude == ['foo*'] + + +def test_cli_without_filename_toplevel(): + '''Modules provided on command line (without files) must be parsed correctly.''' + mods = ['foo', 'foo.bar', 'baz'] + cli_args = argparser().parse_args(mods) + + assert cli_args.filename is None + assert cli_args.top_level is False + assert cli_args.modules == ['foo', 'foo.bar', 'baz'] + + +def test_cli_with_filename_no_cli_mods(): + '''Files (without any modules provided on command line) must be parsed correctly.''' + + files = ['-f', './foo', '-f', './bar', '-f', './baz'] + cli_args = argparser().parse_args(files) + + assert cli_args.filename == [Path('foo'), Path('./bar'), Path('./baz')] + assert not cli_args.top_level + + +def test_main_raises_error_when_no_modules_provided(): + '''If no filename nor modules were provided, ValueError is raised.''' + + with pytest.raises(ValueError): + modules_main([]) + + +def test_import_all_modules_does_not_import(): + '''Ensure the files from /usr/lib/rpm/redhat cannot be imported and + checked for import''' + + # We already imported it in this file once, make sure it's not imported + # from the cache + sys.modules.pop('import_all_modules') + with pytest.raises(ModuleNotFoundError): + modules_main(['import_all_modules']) + + +def test_modules_from_cwd_not_found(tmp_path, monkeypatch): + test_module = tmp_path / 'this_is_a_module_in_cwd.py' + test_module.write_text('') + monkeypatch.chdir(tmp_path) + with pytest.raises(ModuleNotFoundError): + modules_main(['this_is_a_module_in_cwd']) + + +def test_modules_from_sys_path_found(tmp_path): + test_module = tmp_path / 'this_is_a_module_in_sys_path.py' + test_module.write_text('') + sys.path.append(str(tmp_path)) + modules_main(['this_is_a_module_in_sys_path']) + assert 'this_is_a_module_in_sys_path' in sys.modules + + +def test_modules_from_file_are_found(tmp_path): + test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' + test_file.write_text('math\nwave\nsunau\n') + + # Make sure the tested modules are not already in sys.modules + for m in ('math', 'wave', 'sunau'): + sys.modules.pop(m, None) + + modules_main(['-f', str(test_file)]) + + assert 'sunau' in sys.modules + assert 'math' in sys.modules + assert 'wave' in sys.modules + + +def test_modules_from_files_are_found(tmp_path): + test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' + test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' + test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' + + test_file_1.write_text('math\nwave\n') + test_file_2.write_text('sunau\npathlib\n') + test_file_3.write_text('logging\nsunau\n') + + # Make sure the tested modules are not already in sys.modules + for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): + sys.modules.pop(m, None) + + modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) + for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): + assert module in sys.modules + + +def test_nonexisting_modules_raise_exception_on_import(tmp_path): + test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' + test_file.write_text('nonexisting_module\nanother\n') + with pytest.raises(ModuleNotFoundError): + modules_main(['-f', str(test_file)]) + + +def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): + + # This one is supposed to raise an error + cwd_path = tmp_path / 'test_cwd' + Path.mkdir(cwd_path) + test_module_1 = cwd_path / 'this_is_a_module_in_cwd.py' + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_2 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_3 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_4 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3, test_module_4): + module.write_text('') + + sys.path.append(str(tmp_path)) + monkeypatch.chdir(cwd_path) + + with pytest.raises(ModuleNotFoundError): + modules_main([ + 'this_is_a_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + 'this_is_a_module_in_cwd']) + + _, err = capsys.readouterr() + assert 'Check import: this_is_a_module_in_level_0' in err + assert 'Check import: nested.this_is_a_module_in_level_1' in err + assert 'Check import: nested.more_nested.this_is_a_module_in_level_2' in err + assert 'Check import: this_is_a_module_in_cwd' in err + + +def test_modules_both_from_files_and_cli_are_imported(tmp_path): + test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' + test_file_1.write_text('this_is_a_module_in_tmp_path_1') + + test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' + test_file_2.write_text('this_is_a_module_in_tmp_path_2') + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + modules_main([ + '-f', str(test_file_1), + 'this_is_a_module_in_tmp_path_3', + '-f', str(test_file_2), + ]) + + expected = ( + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ) + for module in expected: + assert module in sys.modules + + +def test_non_existing_module_raises_exception(tmp_path): + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + with pytest.raises(ModuleNotFoundError): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + +def test_module_with_error_propagates_exception(tmp_path): + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('0/0') + sys.path.append(str(tmp_path)) + + # The correct exception must be raised + with pytest.raises(ZeroDivisionError): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + ]) + + +def test_correct_modules_are_excluded(tmp_path): + test_module_1 = tmp_path / 'module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt' + test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n') + + modules_main([ + '-e', 'module_in_tmp_path_2', + '-f', str(test_file_1), + '-e', 'module_in_tmp_path_3', + ]) + + assert 'module_in_tmp_path_1' in sys.modules + assert 'module_in_tmp_path_2' not in sys.modules + assert 'module_in_tmp_path_3' not in sys.modules + + +def test_excluding_all_modules_raises_error(tmp_path): + test_module_1 = tmp_path / 'module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt' + test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n') + + with pytest.raises(ValueError): + modules_main([ + '-e', 'module_in_tmp_path*', + '-f', str(test_file_1), + ]) + + +def test_only_toplevel_modules_found(tmp_path): + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + modules_main([ + 'this_is_a_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + '-t']) + + assert 'nested.this_is_a_module_in_level_1' not in sys.modules + assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules + + +def test_only_toplevel_included_modules_found(tmp_path): + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_4 = tmp_path / 'this_is_another_module_in_level_0.py' + + test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3, test_module_4): + module.write_text('') + + sys.path.append(str(tmp_path)) + + modules_main([ + 'this_is_a_module_in_level_0', + 'this_is_another_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + '-t', + '-e', '*another*' + ]) + + assert 'nested.this_is_a_module_in_level_1' not in sys.modules + assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules + assert 'this_is_another_module_in_level_0' not in sys.modules + assert 'this_is_a_module_in_level_0' in sys.modules + + +def test_module_list_from_relative_path(tmp_path, monkeypatch): + + monkeypatch.chdir(tmp_path) + test_file_1 = Path('this_is_a_file_in_cwd.txt') + test_file_1.write_text('wave') + + sys.modules.pop('wave', None) + + modules_main([ + '-f', 'this_is_a_file_in_cwd.txt' + ]) + + assert 'wave' in sys.modules diff --git a/tests/tests.yml b/tests/tests.yml index 1c0478a..11bb3ae 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -15,7 +15,7 @@ tests: - pytest: dir: . - run: ALTERNATE_PYTHON_VERSION=3.6 pytest -v + run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - manual_byte_compilation: dir: . run: rpmbuild -ba pythontest.spec From b20d8aa23aec05a6b6b7b3c55fa3c4b533321b6c Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 1 Nov 2021 09:42:39 +0100 Subject: [PATCH 156/218] Allow multiline arguments processing for %%py3_check_import Fixes the regression introduced with the macro reimplementation. Resolves: rhbz#2018809 --- import_all_modules.py | 4 ++++ macros.python | 7 +++++-- macros.python3 | 7 +++++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 22 +++++++++++++--------- tests/test_import_all_modules.py | 1 + 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/import_all_modules.py b/import_all_modules.py index 7536133..5788b8c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -39,6 +39,10 @@ def read_modules_from_cli(argv): # we need to unify the output to a list of particular elements modules_as_str = ' '.join(argv) modules = re.split(r'[\s,]+', modules_as_str) + # Because of shell expansion in some less typical cases it may happen + # that a trailing space will occur at the end of the list. + # Remove the empty items from the list before passing it further + modules = [m for m in modules if m] return modules diff --git a/macros.python b/macros.python index 6c0ad5b..7233f4e 100644 --- a/macros.python +++ b/macros.python @@ -79,8 +79,11 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} -} + %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ + %{lua: + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) + }} %python_provide() %{lua: local python = require "fedora.srpm.python" diff --git a/macros.python3 b/macros.python3 index d9a4c06..d523eeb 100644 --- a/macros.python3 +++ b/macros.python3 @@ -77,8 +77,11 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} -} + %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ + %{lua: + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) + }} # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 373e389..29b8bef 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 12%{?dist} +Release: 13%{?dist} BuildArch: noarch @@ -145,6 +145,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Nov 01 2021 Karolina Surma - 3.10-13 +- Fix multiline arguments processing for %%py_check_import +Resolves: rhbz#2018809 + * Mon Oct 25 2021 Karolina Surma - 3.10-12 - Introduce -f (read from file) option to %%py{3}_check_import - Introduce -t (filter top-level modules) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index d136cc1..cc6b051 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -671,20 +671,22 @@ def test_python3_sitearch_value_alternate_python(lib, alt_x_y): @pytest.mark.parametrize( - 'args', + 'args, expected_args', [ - 'six', - '-f foo.txt', - '-t -f foo.txt six, seven', - '-e "foo*" -f foo.txt six, seven', - 'six.quarter six.half,, SIX', + ('six', 'six'), + ('-f foo.txt', '-f foo.txt'), + ('-t -f foo.txt six, seven', '-t -f foo.txt six, seven'), + ('-e "foo*" -f foo.txt six, seven', '-e "foo*" -f foo.txt six, seven'), + ('six.quarter six.half,, SIX', 'six.quarter six.half,, SIX'), + ('-f foo.txt six\nsix.half\nSIX', '-f foo.txt six six.half SIX'), + ('six \\ -e six.half', 'six -e six.half'), ] ) @pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/pythonX.Y']) -def test_py3_check_import(args, __python3, lib): +def test_py3_check_import(args, expected_args, __python3, lib): x_y = X_Y macros = { 'buildroot': 'BUILDROOT', @@ -700,7 +702,8 @@ def test_py3_check_import(args, __python3, lib): if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) - lines = rpm_eval(f'%py3_check_import {args}', **macros) + invocation = '%{py3_check_import ' + args +'}' + lines = rpm_eval(invocation, **macros) # An equality check is a bit inflexible here, # every time we change the macro we need to change this test. @@ -711,6 +714,7 @@ def test_py3_check_import(args, __python3, lib): PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {args} + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py + {expected_args} """) assert lines == expected.splitlines() diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 490ef1d..d60f6e4 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -30,6 +30,7 @@ def preserve_sys_modules(): ('five six seven', ['five', 'six', 'seven']), ('six,seven, eight', ['six', 'seven', 'eight']), ('six.quarter six.half,, SIX', ['six.quarter', 'six.half', 'SIX']), + ('six.quarter six.half,, SIX \\ ', ['six.quarter', 'six.half', 'SIX']), ] ) def test_read_modules_from_cli(args, imports): From 824ef3d4afe7c98cea4adea1357e3b1c7b8f6937 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 1 Nov 2021 13:17:42 +0100 Subject: [PATCH 157/218] Fix %%py_shebang_flags handling within %%py_check_import %%py{3}_check_import now respects the custom setting of %%py{3}_shebang_flags and invokes Python with the respective values. If %%py{3}_shebang_flags is undefined or set to no value, there no flags are passed to Python on invoke. Resolves: rhbz#2018615 --- macros.python | 13 ++++++++++--- macros.python3 | 13 ++++++++++--- python-rpm-macros.spec | 2 ++ tests/test_evals.py | 25 +++++++++++++++++++++++-- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/macros.python b/macros.python index 7233f4e..30c6fc3 100644 --- a/macros.python +++ b/macros.python @@ -70,6 +70,7 @@ # With $PATH and $PYTHONPATH set to the %%buildroot, # try to import the Python module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py_shebang_flags or set nothing if it's undefined. # Filter and check import on only top-level modules using -t flag. # Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. @@ -79,11 +80,17 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ %{lua: + local command = "%{__python} " + if rpm.expand("%{?py_shebang_flags}") ~= "" then + command = command .. "-%{py_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 - local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) - }} + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } +} %python_provide() %{lua: local python = require "fedora.srpm.python" diff --git a/macros.python3 b/macros.python3 index d523eeb..c004acf 100644 --- a/macros.python3 +++ b/macros.python3 @@ -68,6 +68,7 @@ # With $PATH and $PYTHONPATH set to the %%buildroot, # try to import the Python 3 module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py3_shebang_flags or set nothing if it's undefined. # Filter and check import on only top-level modules using -t flag. # Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. @@ -77,11 +78,17 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ %{lua: + local command = "%{__python3} " + if rpm.expand("%{?py3_shebang_flags}") ~= "" then + command = command .. "-%{py3_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 - local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) - }} + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } +} # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 29b8bef..f990122 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -148,6 +148,8 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ * Mon Nov 01 2021 Karolina Surma - 3.10-13 - Fix multiline arguments processing for %%py_check_import Resolves: rhbz#2018809 +- Fix %%py_shebang_flags handling within %%py_check_import +Resolves: rhbz#2018615 * Mon Oct 25 2021 Karolina Surma - 3.10-12 - Introduce -f (read from file) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index cc6b051..7cec492 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -691,6 +691,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): macros = { 'buildroot': 'BUILDROOT', '_rpmconfigdir': 'RPMCONFIGDIR', + 'py3_shebang_flags': 's', } if __python3 is not None: if 'X.Y' in __python3: @@ -714,7 +715,27 @@ def test_py3_check_import(args, expected_args, __python3, lib): PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py - {expected_args} + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) assert lines == expected.splitlines() + + +@pytest.mark.parametrize( + 'shebang_flags_value, expected_shebang_flags', + [ + ('s', '-s'), + ('%{nil}', ''), + (None, ''), + ('Es', '-Es'), + ] +) +def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_shebang_flags, lib): + macros = { + '_rpmconfigdir': 'RPMCONFIGDIR', + '__python3': '/usr/bin/python3', + 'py3_shebang_flags': shebang_flags_value, + } + lines = rpm_eval('%py3_check_import sys', **macros) + # Compare the last line of the command, that's where lua part is evaluated + expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' + assert lines[-1].strip() == expected From 9d81ad40e71211a4d307395a14b94dd04cb9af8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Nov 2021 15:53:43 +0100 Subject: [PATCH 158/218] %py(3)_check_import: Process .pth files in site(arch|lib) Fixes https://bugzilla.redhat.com/show_bug.cgi?id=2018551 --- import_all_modules.py | 15 +++++++++++++++ macros.python | 1 + macros.python3 | 1 + python-rpm-macros.spec | 2 ++ tests/test_evals.py | 1 + tests/test_import_all_modules.py | 33 ++++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+) diff --git a/import_all_modules.py b/import_all_modules.py index 5788b8c..3930236 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -3,7 +3,9 @@ import argparse import importlib import fnmatch +import os import re +import site import sys from contextlib import contextmanager @@ -139,6 +141,18 @@ def remove_unwanteds_from_sys_path(): sys.path = old_sys_path +def addsitedirs_from_environ(): + '''Load directories from the _PYTHONSITE environment variable (separated by :) + and load the ones already present in sys.path via site.addsitedir() + to handle .pth files in them. + + This is needed to properly import old-style namespace packages with nspkg.pth files. + See https://bugzilla.redhat.com/2018551 for a more detailed rationale.''' + for path in os.getenv('_PYTHONSITE', '').split(':'): + if path in sys.path: + site.addsitedir(path) + + def main(argv=None): cli_args = argparser().parse_args(argv) @@ -149,6 +163,7 @@ def main(argv=None): modules = read_modules_from_all_args(cli_args) with remove_unwanteds_from_sys_path(): + addsitedirs_from_environ() import_modules(modules) diff --git a/macros.python b/macros.python index 30c6fc3..f0d431d 100644 --- a/macros.python +++ b/macros.python @@ -79,6 +79,7 @@ %py_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{lua: local command = "%{__python} " diff --git a/macros.python3 b/macros.python3 index c004acf..0ccafd9 100644 --- a/macros.python3 +++ b/macros.python3 @@ -77,6 +77,7 @@ %py3_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{lua: local command = "%{__python3} " diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index f990122..aaeddd9 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -150,6 +150,8 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ Resolves: rhbz#2018809 - Fix %%py_shebang_flags handling within %%py_check_import Resolves: rhbz#2018615 +- Process .pth files in buildroot's sitedirs in %%py_check_import +Resolves: rhbz#2018551 * Mon Oct 25 2021 Karolina Surma - 3.10-12 - Introduce -f (read from file) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index 7cec492..6a771ed 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -714,6 +714,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): expected = textwrap.dedent(fr""" PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" + _PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages" PYTHONDONTWRITEBYTECODE=1 {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index d60f6e4..52e1d7e 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -391,3 +391,36 @@ def test_module_list_from_relative_path(tmp_path, monkeypatch): ]) assert 'wave' in sys.modules + + +@pytest.mark.parametrize('arch_in_path', [True, False]) +def test_pth_files_are_read_from__PYTHONSITE(arch_in_path, tmp_path, monkeypatch, capsys): + sitearch = tmp_path / 'lib64' + sitearch.mkdir() + sitelib = tmp_path / 'lib' + sitelib.mkdir() + + for where, word in (sitearch, "ARCH"), (sitelib, "LIB"), (sitelib, "MOD"): + module = where / f'print{word}.py' + module.write_text(f'print("{word}")') + + pth_sitearch = sitearch / 'ARCH.pth' + pth_sitearch.write_text('import printARCH\n') + + pth_sitelib = sitelib / 'LIB.pth' + pth_sitelib.write_text('import printLIB\n') + + if arch_in_path: + sys.path.append(str(sitearch)) + sys.path.append(str(sitelib)) + + # we always add sitearch to _PYTHONSITE + # but when not in sys.path, it should not be processed for .pth files + monkeypatch.setenv('_PYTHONSITE', f'{sitearch}:{sitelib}') + + modules_main(['printMOD']) + out, err = capsys.readouterr() + if arch_in_path: + assert out == 'ARCH\nLIB\nMOD\n' + else: + assert out == 'LIB\nMOD\n' From 9eae0ccaf1f54b46b3c9e1d164a8ea98d0bb20e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Nov 2021 16:04:22 +0100 Subject: [PATCH 159/218] Move import_all_modules out of python-srpm-macros There's no need for it in the default buildroot. --- python-rpm-macros.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index aaeddd9..0f68adc 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -86,7 +86,7 @@ Summary: RPM macros for building Python 3 packages # For %%__python3 and %%python3 Requires: python-srpm-macros = %{version}-%{release} -# For %%py_setup +# For %%py_setup and import_all_modules.py Requires: python-rpm-macros = %{version}-%{release} %description -n python3-rpm-macros @@ -130,11 +130,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{_rpmconfigdir}/redhat/import_all_modules.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py -%{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -152,6 +152,7 @@ Resolves: rhbz#2018809 Resolves: rhbz#2018615 - Process .pth files in buildroot's sitedirs in %%py_check_import Resolves: rhbz#2018551 +- Move import_all_modules.py from python-srpm-macros to python-rpm-macros * Mon Oct 25 2021 Karolina Surma - 3.10-12 - Introduce -f (read from file) option to %%py{3}_check_import From b55e6151bd54255378d0beaaf6107d3fe99ce56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 8 Dec 2021 14:37:09 +0100 Subject: [PATCH 160/218] Move %python3_pkgversion definition earlier in the file So we can use it later in %__python3 and maintain readability. --- macros.python-srpm | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index dcc7023..fcffd23 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -1,22 +1,3 @@ -# Define the Python interpreter paths in the SRPM macros so that -# - they can be used in Build/Requires -# - they can be used in non-Python packages where requiring pythonX-devel would -# be an overkill - -# use the underscored macros to redefine the behavior of %%python3_version etc. -%__python2 /usr/bin/python2 -%__python3 /usr/bin/python3 - -# use the non-underscored macros to refer to Python in spec, etc. -%python2 %__python2 -%python3 %__python3 - -# See https://fedoraproject.org/wiki/Changes/PythonMacroError -%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} - -# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) -%python %__python - # There are multiple Python 3 versions packaged, but only one can be the "main" version # That means that it owns the "python3" namespace: # - python3 package name @@ -49,6 +30,25 @@ # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 +# Define the Python interpreter paths in the SRPM macros so that +# - they can be used in Build/Requires +# - they can be used in non-Python packages where requiring pythonX-devel would +# be an overkill + +# use the underscored macros to redefine the behavior of %%python3_version etc. +%__python2 /usr/bin/python2 +%__python3 /usr/bin/python3 + +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 +%python3 %__python3 + +# See https://fedoraproject.org/wiki/Changes/PythonMacroError +%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} + +# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) +%python %__python + # Define where Python wheels will be stored and the prefix of -wheel packages # - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that # install packages into `/usr/share/python-wheels`. Both names are not From a8b26546eb699afe0dbfcef913a2aa7085fc5afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 8 Dec 2021 12:16:03 +0100 Subject: [PATCH 161/218] Set %__python3 value according to %python3_pkgversion I.e. when %python3_pkgversion is 3.12, %__python3 is /usr/bin/python3.12 We assume that when packagers pacakge for Python 3.X, they want to change both %python3_pkgversion and %__python3 value. Hence instead of copy-pasting this: %global python3_pkgversion 3.X %global __python3 /usr/bin/python3.X They just need to do: %global python3_pkgversion 3.X Packagers who want to change the value of %__python3 without touching %python3_pkgversion can still do it: %global __python3 /usr/bin/pypy3 Related to https://bugzilla.redhat.com/1821489 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 19 +++++++++++++++++-- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index fcffd23..3009f3f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -37,7 +37,7 @@ # use the underscored macros to redefine the behavior of %%python3_version etc. %__python2 /usr/bin/python2 -%__python3 /usr/bin/python3 +%__python3 /usr/bin/python%{python3_pkgversion} # use the non-underscored macros to refer to Python in spec, etc. %python2 %__python2 diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0f68adc..3483cac 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 13%{?dist} +Release: 14%{?dist} BuildArch: noarch @@ -145,6 +145,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Dec 08 2021 Miro Hrončok - 3.10-14 +- Set %%__python3 value according to %%python3_pkgversion + I.e. when %%python3_pkgversion is 3.12, %%__python3 is /usr/bin/python3.12 + * Mon Nov 01 2021 Karolina Surma - 3.10-13 - Fix multiline arguments processing for %%py_check_import Resolves: rhbz#2018809 diff --git a/tests/test_evals.py b/tests/test_evals.py index 6a771ed..ae2b606 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -83,6 +83,17 @@ def shell_stdout(script): shell=True).rstrip() +@pytest.mark.parametrize('macro', ['%__python3', '%python3']) +def test_python3(macro): + assert rpm_eval(macro) == ['/usr/bin/python3'] + + +@pytest.mark.parametrize('macro', ['%__python3', '%python3']) +@pytest.mark.parametrize('pkgversion', ['3', '3.9', '3.12']) +def test_python3_with_pkgversion(macro, pkgversion): + assert rpm_eval(macro, python3_pkgversion=pkgversion) == [f'/usr/bin/python{pkgversion}'] + + @pytest.mark.parametrize('argument, result', [ ('a', 'a'), ('a-a', 'a-a'), @@ -647,7 +658,9 @@ def test_python3_sitelib_value_default(): def test_python3_sitelib_value_alternate_python(alt_x_y): macro = '%python3_sitelib' - assert rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == [f'/usr/lib/python{alt_x_y}/site-packages'] + assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == + rpm_eval(macro, python3_pkgversion=alt_x_y) == + [f'/usr/lib/python{alt_x_y}/site-packages']) def test_python_sitearch_value_python3(lib): @@ -667,7 +680,9 @@ def test_python3_sitearch_value_default(lib): def test_python3_sitearch_value_alternate_python(lib, alt_x_y): macro = '%python3_sitearch' - assert rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == [f'/usr/{lib}/python{alt_x_y}/site-packages'] + assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == + rpm_eval(macro, python3_pkgversion=alt_x_y) == + [f'/usr/{lib}/python{alt_x_y}/site-packages']) @pytest.mark.parametrize( From 5d7727c2aa5ba100865cf2dcec59c3e3c2d0b9fb Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Fri, 26 Nov 2021 12:10:52 +0100 Subject: [PATCH 162/218] Add lua helper functions to make it possible to automatically generate Obsoletes tags --- macros.python-srpm | 11 ++++++ python-rpm-macros.spec | 7 +++- python.lua | 81 +++++++++++++++++++++++++++++------------- 3 files changed, 74 insertions(+), 25 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 3009f3f..0a0846f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -191,6 +191,7 @@ %py_provides() %{lua: local python = require 'fedora.srpm.python' + local rhel = rpm.expand('%{?rhel}') local name = rpm.expand('%1') if name == '%1' then rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}') @@ -204,6 +205,16 @@ for i, provide in ipairs(provides) do print('Provides: ' .. provide .. '\\n') end + -- We only generate these Obsoletes on CentOS/RHEL to provide clean upgrade + -- path, e.g. python3-foo obsoletes python3.9-foo from previous RHEL. + -- In Fedora this is not needed as we don't ship ecosystem packages + -- for alternative Python interpreters. + if rhel ~= '' then + local obsoletes = python.python_altobsoletes(name, evr) + for i, obsolete in ipairs(obsoletes) do + print('Obsoletes: ' .. obsolete .. '\\n') + end + end } %python_extras_subpkg(n:i:f:F) %{expand:%{lua: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3483cac..76d264d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 14%{?dist} +Release: 15%{?dist} BuildArch: noarch @@ -145,6 +145,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Dec 21 2021 Tomas Orsava - 3.10-15 +- Add lua helper functions to make it possible to automatically generate + Obsoletes tags +- Modify the %%py_provides macro to also generate Obsoletes tags on CentOS/RHEL + * Wed Dec 08 2021 Miro Hrončok - 3.10-14 - Set %%__python3 value according to %%python3_pkgversion I.e. when %%python3_pkgversion is 3.12, %%__python3 is /usr/bin/python3.12 diff --git a/python.lua b/python.lua index 8391fe7..bd30a85 100644 --- a/python.lua +++ b/python.lua @@ -2,22 +2,25 @@ -- Determine alternate names provided from the given name. -- Used in pythonname provides generator, python_provide and py_provides. --- There are 2 rules: +-- If only_3_to_3_X is false/nil/unused there are 2 rules: -- python3-foo -> python-foo, python3.X-foo -- python3.X-foo -> python-foo, python3-foo +-- If only_3_to_3_X is true there is only 1 rule: +-- python3-foo -> python3.X-foo -- There is no python-foo -> rule, python-foo packages are version agnostic. -- Returns a table/array with strings. Empty when no rule matched. -local function python_altnames(name) +local function python_altnames(name, only_3_to_3_X) local xy = rpm.expand('%{__default_python3_pkgversion}') local altnames = {} local replaced -- NB: dash needs to be escaped! if name:match('^python3%-') then - for i, prefix in ipairs({'python-', 'python' .. xy .. '-'}) do + local prefixes = only_3_to_3_X and {} or {'python-'} + for i, prefix in ipairs({'python' .. xy .. '-', table.unpack(prefixes)}) do replaced = name:gsub('^python3%-', prefix) table.insert(altnames, replaced) end - elseif name:match('^python' .. xy .. '%-') then + elseif name:match('^python' .. xy .. '%-') and not only_3_to_3_X then for i, prefix in ipairs({'python-', 'python3-'}) do replaced = name:gsub('^python' .. xy .. '%-', prefix) table.insert(altnames, replaced) @@ -27,42 +30,72 @@ local function python_altnames(name) end +local function __python_alttags(name, evr, tag_type) + -- for the "provides" tag_type we want also unversioned provides + local only_3_to_3_X = tag_type ~= "provides" + local operator = tag_type == "provides" and ' = ' or ' < ' + + -- global cache that tells what package NEVRs were already processed for the + -- given tag type + if __python_alttags_beenthere == nil then + __python_alttags_beenthere = {} + end + if __python_alttags_beenthere[tag_type] == nil then + __python_alttags_beenthere[tag_type] = {} + end + __python_alttags_beenthere[tag_type][name .. ' ' .. evr] = true + local alttags = {} + for i, altname in ipairs(python_altnames(name, only_3_to_3_X)) do + table.insert(alttags, altname .. operator .. evr) + end + return alttags +end + -- For any given name and epoch-version-release, return provides except self. -- Uses python_altnames under the hood -- Returns a table/array with strings. local function python_altprovides(name, evr) - -- global cache that tells what provides were already processed - if __python_altnames_provides_beenthere == nil then - __python_altnames_provides_beenthere = {} - end - __python_altnames_provides_beenthere[name .. ' ' .. evr] = true - local altprovides = {} - for i, altname in ipairs(python_altnames(name)) do - table.insert(altprovides, altname .. ' = ' .. evr) - end - return altprovides + return __python_alttags(name, evr, "provides") end +-- For any given name and epoch-version-release, return versioned obsoletes except self. +-- Uses python_altnames under the hood +-- Returns a table/array with strings. +local function python_altobsoletes(name, evr) + return __python_alttags(name, evr, "obsoletes") +end + + +local function __python_alttags_once(name, evr, tag_type) + -- global cache that tells what provides were already processed + if __python_alttags_beenthere == nil + or __python_alttags_beenthere[tag_type] == nil + or __python_alttags_beenthere[tag_type][name .. ' ' .. evr] == nil then + return __python_alttags(name, evr, tag_type) + else + return nil + end +end -- Like python_altprovides but only return something once. -- For each argument can only be used once, returns nil otherwise. -- Previous usage of python_altprovides counts as well. local function python_altprovides_once(name, evr) - -- global cache that tells what provides were already processed - if __python_altnames_provides_beenthere == nil then - __python_altnames_provides_beenthere = {} - end - if __python_altnames_provides_beenthere[name .. ' ' .. evr] == nil then - __python_altnames_provides_beenthere[name .. ' ' .. evr] = true - return python_altprovides(name, evr) - else - return nil - end + return __python_alttags_once(name, evr, "provides") +end + +-- Like python_altobsoletes but only return something once. +-- For each argument can only be used once, returns nil otherwise. +-- Previous usage of python_altobsoletes counts as well. +local function python_altobsoletes_once(name, evr) + return __python_alttags_once(name, evr, "obsoletes") end return { python_altnames = python_altnames, python_altprovides = python_altprovides, + python_altobsoletes = python_altobsoletes, python_altprovides_once = python_altprovides_once, + python_altobsoletes_once = python_altobsoletes_once, } From 5547a87f0b708cec8a13d5b42d726cfa97ca0f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 20 Jan 2022 12:24:10 +0100 Subject: [PATCH 163/218] Add eval tests to RHEL %py_provides Obsoletes functionality --- tests/test_evals.py | 67 +++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/tests/test_evals.py b/tests/test_evals.py index ae2b606..2027a01 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -168,67 +168,92 @@ def test_python_provide_doubleuse(): assert len(set(lines)) == 3 -def test_py_provides_python(): - lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 10]) +def test_py_provides_python(rhel): + lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python-foo = 6-1.fc66' in lines assert len(lines) == 1 -def test_py_provides_whatever(): - lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 12]) +def test_py_provides_whatever(rhel): + lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66', rhel=rhel) assert 'Provides: whatever = 6-1.fc66' in lines assert len(lines) == 1 -def test_py_provides_python3(): - lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 9]) +def test_py_provides_python3(rhel): + lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines - assert len(lines) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 -def test_py_provides_python3_epoched(): - lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 13]) +def test_py_provides_python3_epoched(rhel): + lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines - assert len(lines) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 1:6-1.fc66' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 -def test_py_provides_python3X(): - lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 13]) +def test_py_provides_python3X(rhel): + lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66', rhel=rhel) assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert 'Provides: python3-foo = 6-1.fc66' in lines assert len(lines) == 3 -def test_py_provides_python3X_epoched(): - lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 27]) +def test_py_provides_python3X_epoched(rhel): + lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66', rhel=rhel) assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert len(lines) == 3 -def test_py_provides_doubleuse(): +@pytest.mark.parametrize('rhel', [None, 2]) +def test_py_provides_doubleuse(rhel): lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}', - version='6', release='1.fc66') + version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines - assert len(lines) == 6 - assert len(set(lines)) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines + assert len(lines) == 8 + assert len(set(lines)) == 4 + else: + assert len(lines) == 6 + assert len(set(lines)) == 3 -def test_py_provides_with_evr(): +@pytest.mark.parametrize('rhel', [None, 2]) +def test_py_provides_with_evr(rhel): lines = rpm_eval('%py_provides python3-foo 123', - version='6', release='1.fc66') + version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 123' in lines assert 'Provides: python-foo = 123' in lines assert f'Provides: python{X_Y}-foo = 123' in lines - assert len(lines) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 123' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 def test_python_wheel_pkg_prefix(): From 2ebee9d4cb7f0852f35fd2507418dd53605ffe8e Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 28 Jan 2022 17:04:43 +0100 Subject: [PATCH 164/218] Disable certain rpminspect inspections not relevant to this package --- rpminspect.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 rpminspect.yaml diff --git a/rpminspect.yaml b/rpminspect.yaml new file mode 100644 index 0000000..4589e70 --- /dev/null +++ b/rpminspect.yaml @@ -0,0 +1,7 @@ +# completely disabled inspections: +inspections: + # there is no upstream and the files are changed from time to time + addedfiles: off + changedfiles: off + filesize: off + upstream: off From 99e7d8694cbcfeb9bd4336de232677533a16e9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 31 Jan 2022 12:05:30 +0100 Subject: [PATCH 165/218] Explicitly opt-out from Python name-based provides and obsoletes generators In Koji, python3-rpm-generators are not installed during the build. However, packagers can have them installed locally, in mock or in Copr. This way, we make sure the automatic provides (and obsoletes) do not magically appear only in some environments. Since python3-rpm-macros actually requires python-rpm-macros, the requirement is self-satisfied when the automatic provides are generated. --- python-rpm-macros.spec | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 76d264d..cd4d6af 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 15%{?dist} +Release: 16%{?dist} BuildArch: noarch @@ -58,6 +58,12 @@ BuildArch: noarch # For compileall2.py Requires: python-srpm-macros = %{version}-%{release} +# The packages are called python(3)-(s)rpm-macros +# We never want python3-rpm-macros to provide python-rpm-macros +# We opt out from all Python name-based automatic provides and obsoletes +%undefine __pythonname_provides +%undefine __pythonname_obsoletes + %description This package contains the unversioned Python RPM macros, that most implementations should rely on. @@ -145,6 +151,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Jan 31 2022 Miro Hrončok - 3.10-16 +- Explicitly opt-out from Python name-based provides and obsoletes generators + * Tue Dec 21 2021 Tomas Orsava - 3.10-15 - Add lua helper functions to make it possible to automatically generate Obsoletes tags From e250f28d09787690fca4be5ef0e69b740646c8d5 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Tue, 8 Feb 2022 11:53:45 +0100 Subject: [PATCH 166/218] %py_provides: Do not generate Obsoletes for names containing parentheses This mechanism is already implemented in the old %python_provide macro. --- macros.python-srpm | 11 ++++++++--- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 10 ++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 0a0846f..ad6a0f6 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -210,9 +210,14 @@ -- In Fedora this is not needed as we don't ship ecosystem packages -- for alternative Python interpreters. if rhel ~= '' then - local obsoletes = python.python_altobsoletes(name, evr) - for i, obsolete in ipairs(obsoletes) do - print('Obsoletes: ' .. obsolete .. '\\n') + -- Create Obsoletes only if the name does not end in a parenthesis, + -- as Obsoletes can't include parentheses. + -- This most commonly happens when the name contains an isa. + if (string.sub(name, "-1") ~= ")") then + local obsoletes = python.python_altobsoletes(name, evr) + for i, obsolete in ipairs(obsoletes) do + print('Obsoletes: ' .. obsolete .. '\\n') + end end end } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index cd4d6af..4d55010 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 16%{?dist} +Release: 17%{?dist} BuildArch: noarch @@ -151,6 +151,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Feb 08 2022 Tomas Orsava - 3.10-17 +- %%py_provides: Do not generate Obsoletes for names containing parentheses + * Mon Jan 31 2022 Miro Hrončok - 3.10-16 - Explicitly opt-out from Python name-based provides and obsoletes generators diff --git a/tests/test_evals.py b/tests/test_evals.py index 2027a01..37eb030 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -195,6 +195,16 @@ def test_py_provides_python3(rhel): assert len(lines) == 3 +@pytest.mark.parametrize('rhel', [None, 9]) +def test_py_provides_python3_with_isa(rhel): + lines = rpm_eval('%py_provides python3-foo(x86_64)', version='6', release='1.fc66', rhel=rhel) + assert 'Provides: python3-foo(x86_64) = 6-1.fc66' in lines + assert 'Provides: python-foo(x86_64) = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo(x86_64) = 6-1.fc66' in lines + assert f'Obsoletes: python{X_Y}-foo(x86_64) < 6-1.fc66' not in lines + assert len(lines) == 3 + + @pytest.mark.parametrize('rhel', [None, 13]) def test_py_provides_python3_epoched(rhel): lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66', rhel=rhel) From 9102e29afc357de37ca6cf0e102b54f68bdaf18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 6 Apr 2022 19:08:32 +0200 Subject: [PATCH 167/218] Don't use `! ...` as a check See https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/TFQGD7CSTD5WVKVT3WDIGF5D6DID5NK6/ --- python-rpm-macros.spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4d55010..4f53d4d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -130,7 +130,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %check # no macros in comments -! grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* +grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files From cfa45dfdf3ca4572ada0e02342f57dd2c1d77605 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 12 May 2022 11:43:00 +0200 Subject: [PATCH 168/218] Add a note: Python 3.11+ no longer needs PYTHONHASHSEED=0 Implemented in: https://github.com/python/cpython/pull/27926 We keep using it thou, because this is Python version agnostic. Once we drop support for anything older than 3.11, we can remove it. That'll be around ~2030. Assuming the world still exists by then. --- brp-python-bytecompile | 1 + macros.pybytecompile | 1 + 2 files changed, 2 insertions(+) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index ce73856..c1749ab 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -104,6 +104,7 @@ fi # Disable Python hash seed randomization # This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +# Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) export PYTHONHASHSEED=0 shopt -s nullglob diff --git a/macros.pybytecompile b/macros.pybytecompile index 9dabee0..dd8b495 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -14,6 +14,7 @@ # Setting PYTHONHASHSEED=0 disables Python hash seed randomization # This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +# Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ py2_byte_compile () {\ From 546e9a35446ed57a5a5a8aefc9bc04d04627161b Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Thu, 26 May 2022 23:03:12 -0400 Subject: [PATCH 169/218] Support installing to %{_prefix} other than /usr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pass %{_prefix} to install commands and when determining the sitelib and sitearch variables. https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/KEQMMNJ4HTTHSQLK6P4DJJTVPA36SS3W/ Co-Authored-By: Miro Hrončok --- macros.python | 9 +++++---- macros.python3 | 9 +++++---- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 10 ++++++++++ 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/macros.python b/macros.python index f0d431d..cfe89b6 100644 --- a/macros.python +++ b/macros.python @@ -2,8 +2,9 @@ # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location -%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") -%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") +# platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks +%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") %python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") @@ -47,7 +48,7 @@ %py_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } @@ -58,7 +59,7 @@ } %py_install_wheel() %{expand:\\\ - %{__python} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location + %{__python} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python_sitelib}/*.dist-info %{buildroot}%{python_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/macros.python3 b/macros.python3 index 0ccafd9..3977e7a 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,7 +1,8 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location -%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") -%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") +# platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks +%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") %python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") @@ -45,7 +46,7 @@ %py3_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} + %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } @@ -56,7 +57,7 @@ } %py3_install_wheel() %{expand:\\\ - %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location + %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python3_sitelib}/*.dist-info %{buildroot}%{python3_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4d55010..2ed71fb 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 17%{?dist} +Release: 18%{?dist} BuildArch: noarch @@ -151,6 +151,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Thu May 26 2022 Owen Taylor - 3.10-18 +- Support installing to %%{_prefix} other than /usr + * Tue Feb 08 2022 Tomas Orsava - 3.10-17 - %%py_provides: Do not generate Obsoletes for names containing parentheses diff --git a/tests/test_evals.py b/tests/test_evals.py index 37eb030..2489644 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -698,6 +698,11 @@ def test_python3_sitelib_value_alternate_python(alt_x_y): [f'/usr/lib/python{alt_x_y}/site-packages']) +def test_python3_sitelib_value_alternate_prefix(): + macro = '%python3_sitelib' + assert rpm_eval(macro, _prefix='/app') == [f'/app/lib/python{X_Y}/site-packages'] + + def test_python_sitearch_value_python3(lib): macro = '%python_sitearch' assert rpm_eval(macro, __python='%__python3') == [f'/usr/{lib}/python{X_Y}/site-packages'] @@ -720,6 +725,11 @@ def test_python3_sitearch_value_alternate_python(lib, alt_x_y): [f'/usr/{lib}/python{alt_x_y}/site-packages']) +def test_python3_sitearch_value_alternate_prefix(lib): + macro = '%python3_sitearch' + assert rpm_eval(macro, _prefix='/app') == [f'/app/{lib}/python{X_Y}/site-packages'] + + @pytest.mark.parametrize( 'args, expected_args', [ From b8b5cb92da8974aad9bd231f16b8eb31db7ab14c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Mon, 13 Jun 2022 11:23:37 +0200 Subject: [PATCH 170/218] Python 3.11 https://fedoraproject.org/wiki/Changes/Python3.11 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index ad6a0f6..7104f7f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.10 +%__default_python3_version 3.11 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ed58330..126fdca 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 18%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -151,6 +151,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jun 13 2022 Tomáš Hrnčiar - 3.11-1 +- Update main Python to Python 3.11 +- https://fedoraproject.org/wiki/Changes/Python3.11 + * Thu May 26 2022 Owen Taylor - 3.10-18 - Support installing to %%{_prefix} other than /usr From 4085ef49f2f7759d50835f2139c4c010d3aedb83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 20 Jun 2022 15:53:20 +0200 Subject: [PATCH 171/218] Define %python3_cache_tag / %python_cache_tag, e.g. cpython-311 When reviewing https://src.fedoraproject.org/rpms/pyproject-rpm-macros/pull-request/291 we have discovered that there is no macronized way to get this part of some paths and that packagers need to hardcode it as cpython-%{python3_version_nodots}. This way, we have a standardized macro packagers (and other macros) can use. --- macros.python | 1 + macros.python3 | 2 ++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 24 +++++++++++++++++++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/macros.python b/macros.python index cfe89b6..f060340 100644 --- a/macros.python +++ b/macros.python @@ -10,6 +10,7 @@ %python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") %python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") %python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index 3977e7a..4f2b2d2 100644 --- a/macros.python3 +++ b/macros.python3 @@ -8,6 +8,7 @@ %python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") %python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") %python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s @@ -103,6 +104,7 @@ pyminor = path:match("/python3.(%d+)/") or "*" dirname = path:match("(.*/)") modulename = path:match(".*/([^/]+).py") + -- %%python3_cache_tag is not used here because this macro supports not-installed CPythons print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") end } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 126fdca..15ac23b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -151,6 +151,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jun 20 2022 Miro Hrončok - 3.11-2 +- Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 + * Mon Jun 13 2022 Tomáš Hrnčiar - 3.11-1 - Update main Python to Python 3.11 - https://fedoraproject.org/wiki/Changes/Python3.11 diff --git a/tests/test_evals.py b/tests/test_evals.py index 2489644..7a5c66f 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -72,8 +72,17 @@ def get_alt_x_y(): raise ValueError(f"${env_name} must be X.Y") return alternate_python_version -# We don't use the decorator, to be able to call the function itselef + +def get_alt_xy(): + """ + Same as get_alt_x_y() but without a dot + """ + return get_alt_x_y().replace(".", "") + + +# We don't use decorators, to be able to call the functions directly alt_x_y = pytest.fixture(scope="session")(get_alt_x_y) +alt_xy = pytest.fixture(scope="session")(get_alt_xy) def shell_stdout(script): @@ -638,6 +647,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_platform', '%python_platform_triplet', '%python_ext_suffix', + '%python_cache_tag', '%py_shebang_fix', '%py_build', '%py_build_egg', @@ -676,6 +686,18 @@ def test_ext_suffix(): assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"] +def test_cache_tag(): + assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"] + + +def test_cache_tag_alternate_python(alt_x_y, alt_xy): + assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"] + + +def test_cache_tag_alternate_python3(alt_x_y, alt_xy): + assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"] + + def test_python_sitelib_value_python3(): macro = '%python_sitelib' assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages'] From 4d31ea8034c3f81590489c3271924c7dd133090a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 28 Jun 2022 16:40:22 +0200 Subject: [PATCH 172/218] https://fedoraproject.org/wiki/Changes/PythonSafePath --- macros.python | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 7 ++++++- tests/test_evals.py | 11 ++++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/macros.python b/macros.python index f060340..8cee2d8 100644 --- a/macros.python +++ b/macros.python @@ -13,7 +13,7 @@ %python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py -%py_shbang_opts -s +%py_shbang_opts -s%(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ diff --git a/macros.python3 b/macros.python3 index 4f2b2d2..0fb7ce1 100644 --- a/macros.python3 +++ b/macros.python3 @@ -11,7 +11,7 @@ %python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} -%py3_shbang_opts -s +%py3_shbang_opts -s%(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 15ac23b..087c666 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -151,6 +151,11 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jul 19 2022 Miro Hrončok - 3.11-3 +- Add "P" to %%py3_shbang_opts, %%py3_shbang_opts_nodash, %%py3_shebang_flags + and to %%py_shbang_opts, %%py_shbang_opts_nodash, %%py_shebang_flags +- https://fedoraproject.org/wiki/Changes/PythonSafePath + * Mon Jun 20 2022 Miro Hrončok - 3.11-2 - Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 diff --git a/tests/test_evals.py b/tests/test_evals.py index 7a5c66f..d20d184 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -85,6 +85,11 @@ alt_x_y = pytest.fixture(scope="session")(get_alt_x_y) alt_xy = pytest.fixture(scope="session")(get_alt_xy) +# https://fedoraproject.org/wiki/Changes/PythonSafePath +def safe_path_flag(x_y): + return 'P' if tuple(int(i) for i in x_y.split('.')) >= (3, 11) else '' + + def shell_stdout(script): return subprocess.check_output(script, env={**os.environ, 'LANG': 'C.utf-8'}, @@ -409,7 +414,7 @@ def test_py3_shebang_fix(): def test_py3_shebang_fix_default_shebang_flags(): lines = rpm_eval('%py3_shebang_fix arg1 arg2') lines[-1] = 'echo $shebang_flags' - assert shell_stdout('\n'.join(lines)) == '-kas' + assert shell_stdout('\n'.join(lines)) == f'-kas{safe_path_flag(X_Y)}' def test_py3_shebang_fix_custom_shebang_flags(): @@ -773,7 +778,6 @@ def test_py3_check_import(args, expected_args, __python3, lib): macros = { 'buildroot': 'BUILDROOT', '_rpmconfigdir': 'RPMCONFIGDIR', - 'py3_shebang_flags': 's', } if __python3 is not None: if 'X.Y' in __python3: @@ -798,7 +802,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" _PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} + {__python3 or '/usr/bin/python3'} -s{safe_path_flag(x_y)} RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) assert lines == expected.splitlines() @@ -806,6 +810,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): @pytest.mark.parametrize( 'shebang_flags_value, expected_shebang_flags', [ + ('sP', '-sP'), ('s', '-s'), ('%{nil}', ''), (None, ''), From 4abed5f105028158000c4d105840b1953fec5877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 19 Jul 2022 01:15:20 +0200 Subject: [PATCH 173/218] Use the values of %_py3_shebang_s and %_py3_shebang_P in the shebang opts/flags As proposed in https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/141#comment-109228 And discussed in: - https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/4YD2X7HU5U5DFO3N4FWJLPSKVMKH4VSB/ - https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/4YD2X7HU5U5DFO3N4FWJLPSKVMKH4VSB/ --- macros.python | 4 +++- macros.python3 | 4 +++- tests/test_evals.py | 25 +++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/macros.python b/macros.python index 8cee2d8..77b3793 100644 --- a/macros.python +++ b/macros.python @@ -13,7 +13,9 @@ %python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py -%py_shbang_opts -s%(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_s s +%_py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ diff --git a/macros.python3 b/macros.python3 index 0fb7ce1..06aab31 100644 --- a/macros.python3 +++ b/macros.python3 @@ -11,7 +11,9 @@ %python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} -%py3_shbang_opts -s%(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_s s +%_py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ diff --git a/tests/test_evals.py b/tests/test_evals.py index d20d184..8b95e94 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -423,6 +423,31 @@ def test_py3_shebang_fix_custom_shebang_flags(): assert shell_stdout('\n'.join(lines)) == '-kaEs' +@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s) + lines[-1] = 'echo $shebang_flags' + expected = f'-ka{safe_path_flag(X_Y)}' if safe_path_flag(X_Y) else '-k' + assert shell_stdout('\n'.join(lines)) == expected + + +@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_P(_py3_shebang_P): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_P=_py3_shebang_P) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-kas' + + +@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}']) +@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_sP(_py3_shebang_s, _py3_shebang_P): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', + _py3_shebang_s=_py3_shebang_s, + _py3_shebang_P=_py3_shebang_P) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-k' + + @pytest.mark.parametrize('flags', [None, '%{nil}']) def test_py3_shebang_fix_no_shebang_flags(flags): lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags) From 8847b3750af224d221fe178e26fdc4e6d1f4bd6f Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 22 Jul 2022 22:07:20 +0000 Subject: [PATCH 174/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 087c666..d90b4e8 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -151,6 +151,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 22 2022 Fedora Release Engineering - 3.10-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild + * Tue Jul 19 2022 Miro Hrončok - 3.11-3 - Add "P" to %%py3_shbang_opts, %%py3_shbang_opts_nodash, %%py3_shebang_flags and to %%py_shbang_opts, %%py_shbang_opts_nodash, %%py_shebang_flags From bb334db9f37bfc05f6b12ec784cd05a686bb4a4d Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 25 Oct 2022 10:57:44 +0200 Subject: [PATCH 175/218] Include pathfix.py in this package --- macros.python | 8 +- macros.python3 | 8 +- pathfix.py | 199 +++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 15 +++- tests/test_evals.py | 4 +- 5 files changed, 216 insertions(+), 18 deletions(-) create mode 100644 pathfix.py diff --git a/macros.python b/macros.python index 77b3793..65cf929 100644 --- a/macros.python +++ b/macros.python @@ -19,18 +19,12 @@ %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python_version}.py ]; then - pathfix=/usr/bin/pathfix%{python_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py_shebang_flags}" fi - $pathfix -pni %{__python} $shebang_flags} + %{__python} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 06aab31..c6aa17e 100644 --- a/macros.python3 +++ b/macros.python3 @@ -17,18 +17,12 @@ %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python3_version}.py ]; then - pathfix=/usr/bin/pathfix%{python3_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py3_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py3_shebang_flags}" fi - $pathfix -pni %{__python3} $shebang_flags} + %{__python3} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python3} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/pathfix.py b/pathfix.py new file mode 100644 index 0000000..1d7db3a --- /dev/null +++ b/pathfix.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 + +import sys +import os +from stat import * +import getopt + +err = sys.stderr.write +dbg = err +rep = sys.stdout.write + +new_interpreter = None +preserve_timestamps = False +create_backup = True +keep_flags = False +add_flags = b'' + + +def main(): + global new_interpreter + global preserve_timestamps + global create_backup + global keep_flags + global add_flags + + usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' % + sys.argv[0]) + try: + opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn') + except getopt.error as msg: + err(str(msg) + '\n') + err(usage) + sys.exit(2) + for o, a in opts: + if o == '-i': + new_interpreter = a.encode() + if o == '-p': + preserve_timestamps = True + if o == '-n': + create_backup = False + if o == '-k': + keep_flags = True + if o == '-a': + add_flags = a.encode() + if b' ' in add_flags: + err("-a option doesn't support whitespaces") + sys.exit(2) + if not new_interpreter or not new_interpreter.startswith(b'/') or \ + not args: + err('-i option or file-or-directory missing\n') + err(usage) + sys.exit(2) + bad = 0 + for arg in args: + if os.path.isdir(arg): + if recursedown(arg): bad = 1 + elif os.path.islink(arg): + err(arg + ': will not process symbolic links\n') + bad = 1 + else: + if fix(arg): bad = 1 + sys.exit(bad) + + +def ispython(name): + return name.endswith('.py') + + +def recursedown(dirname): + dbg('recursedown(%r)\n' % (dirname,)) + bad = 0 + try: + names = os.listdir(dirname) + except OSError as msg: + err('%s: cannot list directory: %r\n' % (dirname, msg)) + return 1 + names.sort() + subdirs = [] + for name in names: + if name in (os.curdir, os.pardir): continue + fullname = os.path.join(dirname, name) + if os.path.islink(fullname): pass + elif os.path.isdir(fullname): + subdirs.append(fullname) + elif ispython(name): + if fix(fullname): bad = 1 + for fullname in subdirs: + if recursedown(fullname): bad = 1 + return bad + + +def fix(filename): +## dbg('fix(%r)\n' % (filename,)) + try: + f = open(filename, 'rb') + except IOError as msg: + err('%s: cannot open: %r\n' % (filename, msg)) + return 1 + with f: + line = f.readline() + fixed = fixline(line) + if line == fixed: + rep(filename+': no change\n') + return + head, tail = os.path.split(filename) + tempname = os.path.join(head, '@' + tail) + try: + g = open(tempname, 'wb') + except IOError as msg: + err('%s: cannot create: %r\n' % (tempname, msg)) + return 1 + with g: + rep(filename + ': updating\n') + g.write(fixed) + BUFSIZE = 8*1024 + while 1: + buf = f.read(BUFSIZE) + if not buf: break + g.write(buf) + + # Finishing touch -- move files + + mtime = None + atime = None + # First copy the file's mode to the temp file + try: + statbuf = os.stat(filename) + mtime = statbuf.st_mtime + atime = statbuf.st_atime + os.chmod(tempname, statbuf[ST_MODE] & 0o7777) + except OSError as msg: + err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) + # Then make a backup of the original file as filename~ + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) + # Now move the temp file to the original file + try: + os.rename(tempname, filename) + except OSError as msg: + err('%s: rename failed (%r)\n' % (filename, msg)) + return 1 + if preserve_timestamps: + if atime and mtime: + try: + os.utime(filename, (atime, mtime)) + except OSError as msg: + err('%s: reset of timestamp failed (%r)\n' % (filename, msg)) + return 1 + # Return success + return 0 + + +def parse_shebang(shebangline): + shebangline = shebangline.rstrip(b'\n') + start = shebangline.find(b' -') + if start == -1: + return b'' + return shebangline[start:] + + +def populate_flags(shebangline): + old_flags = b'' + if keep_flags: + old_flags = parse_shebang(shebangline) + if old_flags: + old_flags = old_flags[2:] + if not (old_flags or add_flags): + return b'' + # On Linux, the entire string following the interpreter name + # is passed as a single argument to the interpreter. + # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s" + # so shebang should have single '-' where flags are given and + # flag might need argument for that reasons adding new flags is + # between '-' and original flags + # e.g. #! /usr/bin/python3 -sW Error + return b' -' + add_flags + old_flags + + +def fixline(line): + if not line.startswith(b'#!'): + return line + + if b"python" not in line: + return line + + flags = populate_flags(line) + return b'#! ' + new_interpreter + flags + b'\n' + + +if __name__ == '__main__': + main() diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d90b4e8..8a65425 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -16,6 +16,8 @@ Source201: python.lua %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py +%global pathfix_version 1.0.0 +Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -34,6 +36,7 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT # compileall2.py: PSFv2 +# pathfix.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -49,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -103,6 +106,10 @@ RPM macros for building Python 3 packages. %autosetup -c -T cp -a %{sources} . +# We want to have shebang in the script upstream but not here so +# the package with macros does not depend on Python. +sed -i '1s=^#!/usr/bin/env python3==' pathfix.py + %install mkdir -p %{buildroot}%{rpmmacrodir} @@ -114,7 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ - +install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -137,6 +144,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile %{_rpmconfigdir}/redhat/import_all_modules.py +%{_rpmconfigdir}/redhat/pathfix.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm @@ -151,6 +159,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Oct 25 2022 Lumír Balhar - 3.11-5 +- Include pathfix.py in this package + * Fri Jul 22 2022 Fedora Release Engineering - 3.10-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 8b95e94..94229dc 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -408,7 +408,7 @@ def test_pypi_source_explicit_tilde(): def test_py3_shebang_fix(): cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/python3 -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' def test_py3_shebang_fix_default_shebang_flags(): @@ -457,7 +457,7 @@ def test_py3_shebang_fix_no_shebang_flags(flags): def test_py_shebang_fix_custom_python(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/pypy -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' def test_pycached_in_sitelib(): From 86c391c493085ba7101aa595bf7c017b3dd52d06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Oct 2022 17:34:50 +0200 Subject: [PATCH 176/218] Set PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus} from %pytest pytest-xdist 3+ respects this value when -n auto is used. See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/FQUUM3SZLRY3UUQJ355H4IJS2GRGIEVI/ --- macros.python3 | 1 + python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/macros.python3 b/macros.python3 index c6aa17e..f45af14 100644 --- a/macros.python3 +++ b/macros.python3 @@ -113,4 +113,5 @@ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}\\\ %__pytest} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8a65425..8c4d3f3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -52,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -159,6 +159,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Nov 13 2022 Miro Hrončok - 3.11-6 +- Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest +- pytest-xdist 3+ respects this value when -n auto is used + * Tue Oct 25 2022 Lumír Balhar - 3.11-5 - Include pathfix.py in this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 94229dc..183dcc2 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -319,6 +319,11 @@ def test_pytest_command_suffix_alternate_pkgversion(version): assert f'/usr/bin/pytest-{version} -v' in lines[-1] +def test_pytest_sets_pytest_xdist_auto_num_workers(): + lines = rpm_eval('%pytest', _smp_build_ncpus=2) + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=2' in '\n'.join(lines) + + def test_pytest_undefined_addopts_are_not_set(): lines = rpm_eval('%pytest', __pytest_addopts=None) assert 'PYTEST_ADDOPTS' not in '\n'.join(lines) From b6479253006cef572fb0cbe7ce001b3048f127b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 24 Oct 2022 17:47:59 +0200 Subject: [PATCH 177/218] Expose the environment variables used by %pytest via %{py3_test_envvars} This way, we will be able to reuse them in %tox from pyproject-rpm-macros. Packagers will be able to use them in their spec files. See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/A3QQSHP5OSLIVMCL52AR2GRBRXYQHU6B/ --- macros.python | 10 ++++++++++ macros.python3 | 13 ++++++++----- python-rpm-macros.spec | 1 + tests/test_evals.py | 23 +++++++++++++++++++++++ 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/macros.python b/macros.python index 65cf929..b45c985 100644 --- a/macros.python +++ b/macros.python @@ -131,6 +131,16 @@ end } +# Environment variables for testing used standalone, e.g.: +# %%{py_test_envvars} %%{python} -m unittest +%py_test_envvars %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + %python_disable_dependency_generator() \ %undefine __pythondist_requires \ %{nil} diff --git a/macros.python3 b/macros.python3 index f45af14..6aa541b 100644 --- a/macros.python3 +++ b/macros.python3 @@ -105,13 +105,16 @@ end } -# This is intended for Python 3 only, hence also no Python version in the name. -%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) -%pytest %{expand:\\\ +# Environment variables used by %%pytest, %%tox or standalone, e.g.: +# %%{py3_test_envvars} %%{python3} -m unittest +%py3_test_envvars %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}\\\ - %__pytest} + PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) +%pytest %py3_test_envvars %__pytest diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8c4d3f3..e7844fc 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -162,6 +162,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest - pytest-xdist 3+ respects this value when -n auto is used +- Expose the environment variables used by %%pytest via %%{py3_test_envvars} * Tue Oct 25 2022 Lumír Balhar - 3.11-5 - Include pathfix.py in this package diff --git a/tests/test_evals.py b/tests/test_evals.py index 183dcc2..96ac705 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -357,6 +357,28 @@ def test_pytest_addopts_preserves_envvar(__pytest_addopts): assert 'z--' not in echoed +@pytest.mark.parametrize('__pytest_addopts', ['-X', None]) +def test_py3_test_envvars(lib, __pytest_addopts): + lines = rpm_eval('%{py3_test_envvars}\\\n%{python3} -m unittest', + buildroot='BUILDROOT', + _smp_build_ncpus='3', + __pytest_addopts=__pytest_addopts) + assert all(l.endswith('\\') for l in lines[:-1]) + stripped_lines = [l.strip(' \\') for l in lines] + sitearch = f'BUILDROOT/usr/{lib}/python{X_Y}/site-packages' + sitelib = f'BUILDROOT/usr/lib/python{X_Y}/site-packages' + assert f'PYTHONPATH="${{PYTHONPATH:-{sitearch}:{sitelib}}}"' in stripped_lines + assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines + assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines + assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=3' in stripped_lines + if __pytest_addopts: + assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines + else: + assert 'PYTEST_ADDOPTS' not in ''.join(lines) + assert stripped_lines[-1] == '/usr/bin/python3 -m unittest' + + def test_pypi_source_default_name(): urls = rpm_eval('%pypi_source', name='foo', version='6') @@ -691,6 +713,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%py_install_egg', '%py_install_wheel', '%py_check_import', + '%py_test_envvars', ]) From e3494799f813e6d73646597d7730bf128a4c94ab Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 14 Dec 2022 15:37:02 +0100 Subject: [PATCH 178/218] Test data: use csv which will not be removed from stdlib soon --- tests/test_import_all_modules.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 52e1d7e..71feeff 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -141,15 +141,15 @@ def test_modules_from_sys_path_found(tmp_path): def test_modules_from_file_are_found(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' - test_file.write_text('math\nwave\nsunau\n') + test_file.write_text('math\nwave\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau'): + for m in ('math', 'wave', 'csv'): sys.modules.pop(m, None) modules_main(['-f', str(test_file)]) - assert 'sunau' in sys.modules + assert 'csv' in sys.modules assert 'math' in sys.modules assert 'wave' in sys.modules @@ -160,15 +160,15 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('sunau\npathlib\n') - test_file_3.write_text('logging\nsunau\n') + test_file_2.write_text('csv\npathlib\n') + test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'pathlib', 'logging'): assert module in sys.modules From eb7a4fda28cadd4ae5d374414a57b2620a8fe341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 12 Dec 2022 11:13:46 +0100 Subject: [PATCH 179/218] Bytecompilation: Unset $SOURCE_DATE_EPOCH when %clamp_mtime_to_source_date_epoch is not set https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode --- macros.pybytecompile | 6 +++--- macros.python-srpm | 5 ++++- python-rpm-macros.spec | 12 ++++++++---- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index dd8b495..9e5505b 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -18,7 +18,7 @@ %py_byte_compile()\ py2_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ failure=0\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ @@ -27,13 +27,13 @@ py2_byte_compile () {\ }\ \ py3_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ diff --git a/macros.python-srpm b/macros.python-srpm index 7104f7f..70a04ac 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -73,9 +73,12 @@ ## Should python bytecompilation compile outside python specific directories? ## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 %_python_bytecompile_extra 0 +## Helper macro to unset $SOURCE_DATE_EPOCH if %%clamp_mtime_to_source_date_epoch is not set +## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode +%__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e7844fc..2191707 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -52,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -130,9 +130,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ # It also ensures that: # - our BRPs can execute # - if our BRPs affect this package, we don't need to build it twice -%global __brp_python_bytecompile %{buildroot}%{__brp_python_bytecompile} -%global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink} -%global __brp_fix_pyc_reproducibility %{buildroot}%{__brp_fix_pyc_reproducibility} +%define add_buildroot() %{lua:print((macros[macros[1]]:gsub(macros._rpmconfigdir, macros.buildroot .. macros._rpmconfigdir)))} +%global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} +%global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} +%global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} %check @@ -159,6 +160,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Dec 19 2022 Miro Hrončok - 3.11-7 +- Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set + * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest - pytest-xdist 3+ respects this value when -n auto is used From e4baf5ab7e10485fb909bbb715f3243d14a5e752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Dec 2022 20:24:53 +0100 Subject: [PATCH 180/218] Bytecompilation: Pass --invalidation-mode=timestamp to compileall (Only on Python 3.7+, where it exists and matters.) This will replace patch 328 in Python and fix https://bugzilla.redhat.com/2133850 --- brp-python-bytecompile | 11 ++++++++++- macros.pybytecompile | 14 +++++++++++--- python-rpm-macros.spec | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index c1749ab..347e789 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -47,6 +47,15 @@ function python_bytecompile() compileall_module=compileall2 fi + if [ "$python_version" -ge 37 ]; then + # Force the TIMESTAMP invalidation mode + invalidation_option=--invalidation-mode=timestamp + else + # For older Pythons, the option does not exist + # as the invalidation is always based on size+mtime + invalidation_option= + fi + [ ! -z $exclude ] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib @@ -58,7 +67,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # diff --git a/macros.pybytecompile b/macros.pybytecompile index 9e5505b..9c9da85 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -26,16 +26,21 @@ py2_byte_compile () {\ test $failure -eq 0\ }\ \ -py3_byte_compile () {\ +py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ +py37_byte_compile () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ +}\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ @@ -44,10 +49,13 @@ py39_byte_compile () {\ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ # and it was merged back to stdlib in Python >= 3.9 \ +# Only Python 3.7+ supports and needs the --invalidation-mode option \ if [ "$python_version" -ge 39 ]; then \ py39_byte_compile "%1" "%2"; \ +elif [ "$python_version" -ge 37 ]; then \ +py37_byte_compile "%1" "%2"; \ elif [ "$python_version" -ge 34 ]; then \ -py3_byte_compile "%1" "%2"; \ +py34_byte_compile "%1" "%2"; \ else \ py2_byte_compile "%1" "%2"; \ fi diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 2191707..8ca30d6 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -162,6 +162,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set +- Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From 77912c744dcfd7e4ab7d833686719640bcf45e59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 15 Dec 2022 21:06:17 +0100 Subject: [PATCH 181/218] Add a script to clamp source mtimes, invoke it from the bytecompilation BRP/macro https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes --- brp-python-bytecompile | 15 ++++ clamp_source_mtime.py | 161 +++++++++++++++++++++++++++++++++++++++++ macros.pybytecompile | 8 ++ python-rpm-macros.spec | 6 +- 4 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 clamp_source_mtime.py diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 347e789..3092bdf 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -16,6 +16,16 @@ if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi +# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +function python_clamp_source_mtime() +{ + local _=$1 + local python_binary=$2 + local _=$3 + local python_libdir="$4" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir" +} + # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. @@ -123,6 +133,11 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + # One or more of the files had inaccessible mtime + exit 1 + fi python_bytecompile "" "$python_binary" "" "$python_libdir" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py new file mode 100644 index 0000000..7349ea3 --- /dev/null +++ b/clamp_source_mtime.py @@ -0,0 +1,161 @@ +"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH + +When called as a script with arguments, this compiles the directories +given as arguments recursively. + +If upstream is interested, this can be later integrated to the compileall module +as an additional option (e.g. --clamp-source-mtime). + +License: +This has been derived from the Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ +""" +import os +import sys + +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) + +__all__ = ["clamp_dir", "clamp_file"] + + +def _walk_dir(dir, maxlevels, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) + + +def clamp_dir(dir, source_date_epoch, quiet=0): + """Clamp the mtime of all modules in the given directory tree. + + Arguments: + + dir: the directory to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + maxlevels = sys.getrecursionlimit() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + for file in files: + if not clamp_file(file, source_date_epoch, quiet=quiet): + success = False + return success + + +def clamp_file(fullname, source_date_epoch, quiet=0): + """Clamp the mtime of one file. + + Arguments: + + fullname: the file to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + if os.path.isfile(fullname) and not os.path.islink(fullname): + if name[-3:] == '.py': + try: + mtime = int(os.stat(fullname).st_mtime) + atime = int(os.stat(fullname).st_atime) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error checking mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + if mtime > source_date_epoch: + if not quiet: + print('Clamping mtime of {!r}'.format(fullname)) + try: + os.utime(fullname, (atime, source_date_epoch)) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error clamping mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + return True + + +def main(): + """Script main program.""" + import argparse + + source_date_epoch = os.getenv('SOURCE_DATE_EPOCH') + if not source_date_epoch: + print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set") + return True # This is a success, no action needed + try: + source_date_epoch = int(source_date_epoch) + except ValueError: + print("$SOURCE_DATE_EPOCH must be an integer") + return False + + parser = argparse.ArgumentParser( + description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+', + help=('zero or more file and directory paths ' + 'to clamp')) + + args = parser.parse_args() + clamp_dests = args.clamp_dest + + success = True + try: + for dest in clamp_dests: + if os.path.isfile(dest): + if not clamp_file(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + else: + if not clamp_dir(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + return success + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/macros.pybytecompile b/macros.pybytecompile index 9c9da85..8c9fbee 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -17,6 +17,12 @@ # Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ +clamp_source_mtime () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \ +}\ +\ py2_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ @@ -45,6 +51,8 @@ py39_byte_compile () {\ \ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ +# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \ +clamp_source_mtime "%1" "%2"; \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8ca30d6..d9daf49 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -18,6 +18,7 @@ Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_ Source302: import_all_modules.py %global pathfix_version 1.0.0 Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py +Source304: clamp_source_mtime.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -35,7 +36,7 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py: PSFv2 +# compileall2.py, clamp_source_mtime.py: PSFv2 # pathfix.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -120,6 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -150,6 +152,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/clamp_source_mtime.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -163,6 +166,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) +- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From cb1dbcc44b105f3deadc99f0b79b7711bc0ff6d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 14:38:05 +0100 Subject: [PATCH 182/218] Add Python 2.7 support to clamp_source_mtime.py This script is also invoked with Python 2 --- clamp_source_mtime.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py index 7349ea3..1d03a6b 100644 --- a/clamp_source_mtime.py +++ b/clamp_source_mtime.py @@ -10,6 +10,7 @@ License: This has been derived from the Python's compileall module and it follows Python licensing. For more info see: https://www.python.org/psf/license/ """ +from __future__ import print_function import os import sys @@ -41,8 +42,9 @@ def _walk_dir(dir, maxlevels, quiet=0): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): - yield from _walk_dir(fullname, maxlevels=maxlevels - 1, - quiet=quiet) + for result in _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet): + yield result def clamp_dir(dir, source_date_epoch, quiet=0): From 18ceb6caef878e0de8b22297a63161423653684d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 14:42:54 +0100 Subject: [PATCH 183/218] CI tests: Build pythontest.spec with and without %clamp_mtime_to_source_date_epoch --- tests/pythontest.spec | 5 +++++ tests/tests.yml | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 3b707c5..50c136a 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -71,3 +71,8 @@ test $PY -ge $INODES %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py %{python27_sitelib}/directory/file.py* + + +%changelog +* Thu Jan 01 2015 Fedora Packager - 0-0 +- This changelog entry exists and is deliberately set in the past diff --git a/tests/tests.yml b/tests/tests.yml index 11bb3ae..ac03d2e 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -16,9 +16,12 @@ - pytest: dir: . run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation: + - manual_byte_compilation_clamp_mtime_off: dir: . - run: rpmbuild -ba pythontest.spec + run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - manual_byte_compilation_clamp_mtime_on: + dir: . + run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec required_packages: - rpm-build - python-rpm-macros From 13c1c0c519fd78ced17f790149f5c050015dce45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 15:06:03 +0100 Subject: [PATCH 184/218] CI tests: Assert rpmlint's python-bytecode-inconsistent-mtime is not happening --- tests/pythontest.spec | 2 +- tests/tests.yml | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 50c136a..32bb039 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -7,7 +7,7 @@ Name: pythontest Version: 0 -Release: 0 +Release: 0%{?dist} Summary: ... License: MIT BuildRequires: python3-devel diff --git a/tests/tests.yml b/tests/tests.yml index ac03d2e..4b8b2dd 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -18,12 +18,19 @@ run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - manual_byte_compilation_clamp_mtime_off: dir: . - run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - manual_byte_compilation_clamp_mtime_on: dir: . - run: rpmbuild --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - rpmlint_clamp_mtime_off: + dir: . + run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - rpmlint_clamp_mtime_on: + dir: . + run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 required_packages: - rpm-build + - rpmlint - python-rpm-macros - python3-rpm-macros - python3-devel From a3ea23bd5ed0226fd03d49dc200939bfc778d73b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 19 Dec 2022 15:14:32 +0100 Subject: [PATCH 185/218] Bytecompilation: Compile Python files in parallel, according to %_smp_mflags --- brp-python-bytecompile | 11 +++++++---- macros.pybytecompile | 6 +++--- macros.python-srpm | 2 +- python-rpm-macros.spec | 1 + 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 3092bdf..1f7c4cd 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -11,6 +11,8 @@ if [ 0$extra -eq 1 ]; then exit 1 fi +compileall_flags="$4" + # If using normal root, avoid changing anything. if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 @@ -36,6 +38,7 @@ function python_bytecompile() local python_binary=$2 local exclude=$3 local python_libdir="$4" + local compileall_flags="$5" python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") @@ -77,7 +80,7 @@ function python_bytecompile() # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -133,12 +136,12 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. - python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had inaccessible mtime exit 1 fi - python_bytecompile "" "$python_binary" "" "$python_libdir" + python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 @@ -146,7 +149,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing - python_bytecompile "-O" "$python_binary" "" "$python_libdir" + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 diff --git a/macros.pybytecompile b/macros.pybytecompile index 8c9fbee..e5e5a47 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -35,18 +35,18 @@ py2_byte_compile () {\ py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ py37_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + $python_binary -s -B -m compileall %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ diff --git a/macros.python-srpm b/macros.python-srpm index 70a04ac..5e57102 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -78,7 +78,7 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_mflags}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d9daf49..919f470 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -167,6 +167,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) - Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +- Bytecompilation: Compile Python files in parallel, according to %%_smp_mflags * Sun Nov 13 2022 Miro Hrončok - 3.11-6 - Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest From 9a36534a957594f370f269b739e28dd23b4de66b Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 20 Jan 2023 17:04:00 +0000 Subject: [PATCH 186/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 919f470..58a7704 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + * Mon Dec 19 2022 Miro Hrončok - 3.11-7 - Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set - Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) From 3ca74ad94c19d316938a27cd5758b4ed5053c3a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 6 Jan 2023 15:04:29 +0100 Subject: [PATCH 187/218] Memoize values of macros that execute python to get their value Macros like %{python3_sitelib} were evaluated at every instance in the spec (each time, Python was started, sysconfig was imported...). When there were many instances (40+), it might have taken more than a minute to parse the spec file. This way, the macros are defined via %global on first usage. Every repetitive usage reuses the actual value. --- macros.python | 18 +++++++++--------- macros.python3 | 16 ++++++++-------- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 14 ++++++++++++-- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/macros.python b/macros.python index b45c985..4568614 100644 --- a/macros.python +++ b/macros.python @@ -3,18 +3,18 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") +%python_sitelib %{global python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitelib} +%python_sitearch %{global python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitearch} +%python_version %{global python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python_version} +%python_version_nodots %{global python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python_version_nodots} +%python_platform %{global python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")}%{python_platform} +%python_platform_triplet %{global python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python_platform_triplet} +%python_ext_suffix %{global python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python_ext_suffix} +%python_cache_tag %{global python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")}%{python_cache_tag} %py_setup setup.py %_py_shebang_s s -%_py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_P %{global _py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py_shebang_P} %py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) diff --git a/macros.python3 b/macros.python3 index 6aa541b..0e4678e 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,18 +1,18 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitelib %{global python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python3_sitelib} %python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") +%python3_version %{global python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python3_version} +%python3_version_nodots %{global python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python3_version_nodots} +%python3_platform %{global python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")}%{python3_platform} +%python3_platform_triplet %{global python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python3_platform_triplet} +%python3_ext_suffix %{global python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python3_ext_suffix} +%python3_cache_tag %{global python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")}%{python3_cache_tag} %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %_py3_shebang_s s -%_py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_P %{global _py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py3_shebang_P} %py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 58a7704..abf5ba1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jan 20 2023 Miro Hrončok - 3.11-9 +- Memoize values of macros that execute python to get their value +- Fixes: rhbz#2155505 + * Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 - Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 96ac705..294faaa 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -720,8 +720,18 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) - assert lines == ['error: attempt to use unversioned python, ' - 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'] + assert lines[0] == ( + 'error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' + ) + # when the macros are %global, the error is longer + # we deliberately allow this extra line to be optional + if len(lines) > 1: + # the failed macro is not unnecessarily our tested macro + pattern = r'error: Macro %\S+ failed to expand' + assert re.match(pattern, lines[1]) + # but there should be no more lines + assert len(lines) < 3 @unversioned_macros From 10e0e5309d691e16d2e86d329827f47993879bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Mar 2023 19:43:17 +0100 Subject: [PATCH 188/218] Don't assume %_smp_mflags only ever contains -jX, use -j%_smp_build_ncpus directly When we added %_smp_mflags here, Petr Viktorin asked the question: https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/154#comment-124613 > I couldn't find docs for %_smp_mflags. > How much of a guarantee is there that it contains no other flags than -j? My answer was: > %_smp_mflags is documented in https://rpm-packaging-guide.github.io/ > and used in many other RPM macros in Fedora and upstream everywhere. > There is no official guarantee that it will never contain anything else, > but if it does, I assume multiple things would burn. > I am willing to take that risk. Turns out, the world did not burn, but packagers do set %_smp_mflags to -lX, which does not work with compileall. Fixes https://bugzilla.redhat.com/2179149 --- macros.pybytecompile | 6 +++--- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/macros.pybytecompile b/macros.pybytecompile index e5e5a47..e81ddae 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -35,18 +35,18 @@ py2_byte_compile () {\ py34_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ py37_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall %{?_smp_mflags} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + $python_binary -s -B -m compileall %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ diff --git a/macros.python-srpm b/macros.python-srpm index 5e57102..37cf09a 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -78,7 +78,7 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_mflags}" +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index abf5ba1..b53f7df 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 10%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Mar 16 2023 Miro Hrončok - 3.11-10 +- Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly +- Fixes: rhbz#2179149 + * Fri Jan 20 2023 Miro Hrončok - 3.11-9 - Memoize values of macros that execute python to get their value - Fixes: rhbz#2155505 From 3c21a3a25818ac1512c891b0d3ab79d8fbcaa30e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Tue, 13 Jun 2023 14:43:51 +0200 Subject: [PATCH 189/218] Python 3.12 https://fedoraproject.org/wiki/Changes/Python3.12 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 37cf09a..278571f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.11 +%__default_python3_version 3.12 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b53f7df..347fb4e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 10%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -163,6 +163,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 +- Update main Python to Python 3.12 +- https://fedoraproject.org/wiki/Changes/Python3.12 + * Thu Mar 16 2023 Miro Hrončok - 3.11-10 - Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly - Fixes: rhbz#2179149 From f6368ebcaa89b656bbf95f266ed83c841fbbc77d Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 21 Jul 2023 13:45:20 +0000 Subject: [PATCH 190/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild Signed-off-by: Fedora Release Engineering --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 347fb4e..3948428 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + * Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 - Update main Python to Python 3.12 - https://fedoraproject.org/wiki/Changes/Python3.12 From 933da64fe645d29d8782b7c4df613bdcb2c3b6a8 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 9 Aug 2023 11:08:17 +0200 Subject: [PATCH 191/218] Declare the license as an SPDX expression --- python-rpm-macros.spec | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 3948428..8220091 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -36,10 +36,10 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py, clamp_source_mtime.py: PSFv2 -# pathfix.py: PSFv2 -# brp scripts: GPLv2+ -License: MIT and Python and GPLv2+ +# compileall2.py, clamp_source_mtime.py: PSF-2.0 +# pathfix.py: PSF-2.0 +# brp scripts: GPL-2.0-or-later +License: MIT AND PSF-2.0 AND GPL-2.0-or-later # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed Aug 09 2023 Karolina Surma - 3.11-3 +- Declare the license as an SPDX expression + * Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild From d4a26d93d0939ab246c2fd894c66d4c3a6c0ed9c Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Tue, 5 Sep 2023 00:42:40 -0500 Subject: [PATCH 192/218] Remove %py3_build_egg and %py3_install_egg macros. %py3_install_egg is nonfunctional; setuptools removed the easy_install entrypoint years ago. %py3_build_egg is technically functional but has been superseded by newer macros. Calling setup.py directly is deprecated, and building/installing eggs to begin with is deprecated. The macros are not used in any Fedora packages and are broken. It's time to remove them. --- macros.python | 11 ----------- macros.python3 | 11 ----------- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 2 -- 4 files changed, 4 insertions(+), 25 deletions(-) diff --git a/macros.python b/macros.python index 4568614..e058fff 100644 --- a/macros.python +++ b/macros.python @@ -33,11 +33,6 @@ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - %py_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} @@ -49,12 +44,6 @@ rm -rfv %{buildroot}%{_bindir}/__pycache__ } -%py_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python_sitelib} - %{__python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - %py_install_wheel() %{expand:\\\ %{__python} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python3 b/macros.python3 index 0e4678e..06a7db2 100644 --- a/macros.python3 +++ b/macros.python3 @@ -31,11 +31,6 @@ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - %py3_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} @@ -47,12 +42,6 @@ rm -rfv %{buildroot}%{_bindir}/__pycache__ } -%py3_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python3_sitelib} - %{__python3} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - %py3_install_wheel() %{expand:\\\ %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8220091..708cd30 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Sep 05 2023 Maxwell G - 3.12-4 +- Remove %%py3_build_egg and %%py3_install_egg macros. + * Wed Aug 09 2023 Karolina Surma - 3.11-3 - Declare the license as an SPDX expression diff --git a/tests/test_evals.py b/tests/test_evals.py index 294faaa..2e0e6e1 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -707,10 +707,8 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_cache_tag', '%py_shebang_fix', '%py_build', - '%py_build_egg', '%py_build_wheel', '%py_install', - '%py_install_egg', '%py_install_wheel', '%py_check_import', '%py_test_envvars', From c765382ec8ef1f9c67e1745bc03c9374ddc6a15a Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Fri, 1 Sep 2023 16:22:10 +0200 Subject: [PATCH 193/218] Fix the changelog entries --- python-rpm-macros.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 708cd30..322250d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -166,10 +166,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true * Tue Sep 05 2023 Maxwell G - 3.12-4 - Remove %%py3_build_egg and %%py3_install_egg macros. -* Wed Aug 09 2023 Karolina Surma - 3.11-3 +* Wed Aug 09 2023 Karolina Surma - 3.12-3 - Declare the license as an SPDX expression -* Fri Jul 21 2023 Fedora Release Engineering - 3.11-2 +* Fri Jul 21 2023 Fedora Release Engineering - 3.12-2 - Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild * Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 From 5eec3f76022eb061a1a8d2488d48ec3ab477d04c Mon Sep 17 00:00:00 2001 From: Maxwell G Date: Wed, 31 May 2023 17:05:04 -0500 Subject: [PATCH 194/218] Fix python macro memoizing to account for changing %__python3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This adds a new `%_python_memoize` macro that caches the values of various python macros in a lua table based on the current value of `%__python` / `%__python3`. The Python macros are adjusted to use this macro for memoization instead of the current approach. This way, the macros will return values that apply to the _current_ `%__python3` value when packagers create multi-python specfiles that toggle the value of `%python3_pkgversion`. Relates: https://bugzilla.redhat.com/2209055 Co-Authored-By: Miro Hrončok --- macros.python | 59 +++++++++++++++++++++++++++++++++++------- macros.python3 | 35 ++++++++++++++++++------- python-rpm-macros.spec | 5 +++- tests/test_evals.py | 35 +++++++++++++++++++++++-- 4 files changed, 113 insertions(+), 21 deletions(-) diff --git a/macros.python b/macros.python index e058fff..0d29ae0 100644 --- a/macros.python +++ b/macros.python @@ -1,20 +1,61 @@ +# Memoize a macro to avoid calling the same expensive code multiple times in +# the specfile. +# There is no error handling, +# memoizing an undefined macro (or using such a key) has undefined behavior. +# Options: +# -n - The name of the macro to wrap +# -k - The name of the macro to use as a cache key +%_python_memoize(n:k:) %{lua: +local name = opt.n +-- NB: We use rpm.expand() here instead of the macros table to make sure errors +-- are propogated properly. +local cache_key = rpm.expand("%{" .. opt.k .. "}") +if not _python_macro_cache then + -- This is intentionally a global lua table + _python_macro_cache = {} +end +if not _python_macro_cache[cache_key] then + _python_macro_cache[cache_key] = {} +end +if not _python_macro_cache[cache_key][name] then + _python_macro_cache[cache_key][name] = rpm.expand("%{" .. name .. "}") +end +print(_python_macro_cache[cache_key][name]) +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python_sitelib %{global python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitelib} -%python_sitearch %{global python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python_sitearch} -%python_version %{global python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python_version} -%python_version_nodots %{global python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python_version_nodots} -%python_platform %{global python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")}%{python_platform} -%python_platform_triplet %{global python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python_platform_triplet} -%python_ext_suffix %{global python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python_ext_suffix} -%python_cache_tag %{global python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")}%{python_cache_tag} +%__python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitelib %{_python_memoize -n __python_sitelib -k __python} + +%__python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitearch %{_python_memoize -n __python_sitearch -k __python} + +%__python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version %{_python_memoize -n __python_version -k __python} + +%__python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_version_nodots %{_python_memoize -n __python_version_nodots -k __python} + +%__python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform %{_python_memoize -n __python_platform -k __python} + +%__python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_platform_triplet %{_python_memoize -n __python_platform_triplet -k __python} + +%__python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_ext_suffix %{_python_memoize -n __python_ext_suffix -k __python} + +%__python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") +%python_cache_tag %{_python_memoize -n __python_cache_tag -k __python} %py_setup setup.py %_py_shebang_s s -%_py_shebang_P %{global _py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py_shebang_P} +%__py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_P %{_python_memoize -n __py_shebang_P -k __python} %py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) diff --git a/macros.python3 b/macros.python3 index 06a7db2..a813557 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,18 +1,35 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%python3_sitelib %{global python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))")}%{python3_sitelib} -%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_version %{global python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))")}%{python3_version} -%python3_version_nodots %{global python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))")}%{python3_version_nodots} -%python3_platform %{global python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")}%{python3_platform} -%python3_platform_triplet %{global python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")}%{python3_platform_triplet} -%python3_ext_suffix %{global python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")}%{python3_ext_suffix} -%python3_cache_tag %{global python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")}%{python3_cache_tag} +%__python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitelib %{_python_memoize -n __python3_sitelib -k __python3} + +%__python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitearch %{_python_memoize -n __python3_sitearch -k __python3} + +%__python3_version %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version %{_python_memoize -n __python3_version -k __python3} + +%__python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_version_nodots %{_python_memoize -n __python3_version_nodots -k __python3} + +%__python3_platform %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python3_platform %{_python_memoize -n __python3_platform -k __python3} + +%__python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_platform_triplet %{_python_memoize -n __python3_platform_triplet -k __python3} + +%__python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_ext_suffix %{_python_memoize -n __python3_ext_suffix -k __python3} + +%__python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; print(sys.implementation.cache_tag)") +%python3_cache_tag %{_python_memoize -n __python3_cache_tag -k __python3} + %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %_py3_shebang_s s -%_py3_shebang_P %{global _py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')")}%{_py3_shebang_P} +%__py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_P %{_python_memoize -n __py3_shebang_P -k __python3} %py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 322250d..e8f7a4c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Oct 09 2023 Maxwell G - 3.12-5 +- Fix python macro memoizing to account for changing %%__python3 + * Tue Sep 05 2023 Maxwell G - 3.12-4 - Remove %%py3_build_egg and %%py3_install_egg macros. diff --git a/tests/test_evals.py b/tests/test_evals.py index 2e0e6e1..69f689e 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -22,6 +22,8 @@ TESTED_FILES = os.getenv("TESTED_FILES", None) def rpm_eval(expression, fails=False, **kwargs): + if isinstance(expression, str): + expression = [expression] cmd = ['rpmbuild'] if TESTED_FILES: cmd += ['--macros', TESTED_FILES] @@ -30,7 +32,8 @@ def rpm_eval(expression, fails=False, **kwargs): cmd += ['--undefine', var] else: cmd += ['--define', f'{var} {value}'] - cmd += ['--eval', expression] + for e in expression: + cmd += ['--eval', e] cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if fails: @@ -724,7 +727,7 @@ def test_unversioned_python_errors(macro): ) # when the macros are %global, the error is longer # we deliberately allow this extra line to be optional - if len(lines) > 1: + if len(lines) > 1 and "error: lua script failed" not in lines[1]: # the failed macro is not unnecessarily our tested macro pattern = r'error: Macro %\S+ failed to expand' assert re.match(pattern, lines[1]) @@ -888,3 +891,31 @@ def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_s # Compare the last line of the command, that's where lua part is evaluated expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' assert lines[-1].strip() == expected + + +def test_multi_python(alt_x_y): + """ + Ensure memoized %python_version works when switching %__python back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python /usr/bin/python{version}', '%python_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +def test_multi_python3(alt_x_y): + """ + Ensure memoized %python3_version works when switching %__python3 back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python3 /usr/bin/python{version}', '%python3_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] From d1c3ea93f88d6d4e83489373727104d322f00d42 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Mon, 22 Jan 2024 06:41:47 +0000 Subject: [PATCH 195/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e8f7a4c..8b5fba3 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + * Mon Oct 09 2023 Maxwell G - 3.12-5 - Fix python macro memoizing to account for changing %%__python3 From b2f798fc819c1c0b3254d6cd4d24e04dd97e49cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 18 Jan 2023 16:36:05 +0100 Subject: [PATCH 196/218] %py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set --- macros.python | 2 +- macros.python3 | 2 +- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/macros.python b/macros.python index 0d29ae0..0ba4d78 100644 --- a/macros.python +++ b/macros.python @@ -169,7 +169,7 @@ print(_python_macro_cache[cache_key][name]) PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} %python_disable_dependency_generator() \ %undefine __pythondist_requires \ diff --git a/macros.python3 b/macros.python3 index a813557..7197433 100644 --- a/macros.python3 +++ b/macros.python3 @@ -119,7 +119,7 @@ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS=%{_smp_build_ncpus}} + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} # This is intended for Python 3 only, hence also no Python version in the name. %__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8b5fba3..06fc622 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Jan 25 2024 Miro Hrončok - 3.12-7 +- %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set + * Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 - Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 69f689e..1f953e0 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -324,7 +324,7 @@ def test_pytest_command_suffix_alternate_pkgversion(version): def test_pytest_sets_pytest_xdist_auto_num_workers(): lines = rpm_eval('%pytest', _smp_build_ncpus=2) - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=2' in '\n'.join(lines) + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-2}"' in '\n'.join(lines) def test_pytest_undefined_addopts_are_not_set(): @@ -374,7 +374,7 @@ def test_py3_test_envvars(lib, __pytest_addopts): assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS=3' in stripped_lines + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-3}"' in stripped_lines if __pytest_addopts: assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines else: From 2acd7cb9b37be66ac1fe1d85134698d66b6ae170 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Thu, 21 Mar 2024 17:02:42 +0100 Subject: [PATCH 197/218] Update bundled compileall2 to version 0.8.0 --- compileall2.py | 80 +++++++++++++++++++++++------------------- python-rpm-macros.spec | 7 ++-- 2 files changed, 48 insertions(+), 39 deletions(-) diff --git a/compileall2.py b/compileall2.py index c58e545..ea7e76f 100644 --- a/compileall2.py +++ b/compileall2.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, if compiles all modules on sys.path, without +Without arguments, it compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -36,11 +36,11 @@ PY35 = sys.version_info[0:2] >= (3, 5) # introduced in Python 3.7. These cases are covered by variables here or by PY37 # variable itself. if PY37: - pyc_struct_format = '<4sll' + pyc_struct_format = '<4sLL' pyc_header_lenght = 12 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) else: - pyc_struct_format = '<4sl' + pyc_struct_format = '<4sL' pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) @@ -106,7 +106,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path @@ -120,23 +120,34 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, stripdir = dir prependdir = ddir ddir = None - if workers is not None: - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - elif workers != 1: - try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: - workers = 1 + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits + try: + _check_system_limits() + except NotImplementedError: + workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True - if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + if workers != 1 and ProcessPoolExecutor is not None: + mp_context_arg = {} + if PY37: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + mp_context_arg = {"mp_context": mp_context} + # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + **mp_context_arg) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -178,7 +189,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. @@ -190,10 +201,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) - else: - fullname = str(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None @@ -206,13 +215,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: @@ -258,7 +267,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack(*(pyc_header_format + (mtime,))) + expect = struct.pack(*(pyc_header_format + (mtime & 0xFFFF_FFFF,))) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(pyc_header_lenght) @@ -301,9 +310,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) + encoding = sys.stdout.encoding or sys.getdefaultencoding() + msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False @@ -408,8 +416,8 @@ def main(): type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses optimization level of ' - 'Python interpreter itself (specified by -O).')) + 'Default is -1 which uses the optimization level ' + 'of the Python interpreter itself (see -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', @@ -456,7 +464,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: @@ -464,9 +473,6 @@ def main(): print("Error reading file list {}".format(args.flist)) return False - if args.workers is not None: - args.workers = args.workers or None - if PY37 and args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 06fc622..a3c2511 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -13,7 +13,7 @@ Source105: macros.pybytecompile Source201: python.lua # Python code -%global compileall2_version 0.7.1 +%global compileall2_version 0.8.0 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py %global pathfix_version 1.0.0 @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Mar 22 2024 Lumír Balhar - 3.12-8 +- Update bundled compileall2 to version 0.8.0 + * Thu Jan 25 2024 Miro Hrončok - 3.12-7 - %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set From d211646a5e0cc32f62efe963d7970b896445f9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= Date: Fri, 1 Mar 2024 00:27:31 +0100 Subject: [PATCH 198/218] brp-fix-pyc-reproducibility: use more strict shell style 1. Error out on unset variables. 2. Implement suggestion made by shellcheck: use "||" instead of "-o"" in test. Shellcheck also suggested -print0 to avoid ambiguity with filenames with embedded newlines, but instead use '-exec {} +', which doesn't need xargs but still passes multiple arguments to a single marshalparser invocation. (Those issues are unlikely to cause problems in the rpm environment, but it's nice to be shellcheck-clean to use shellcheck during development.) Co-Authored-By: Benjamin A. Beasley --- brp-fix-pyc-reproducibility | 10 ++++------ python-rpm-macros.spec | 5 ++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 536a126..16613b0 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,13 +1,13 @@ -#!/bin/bash -e +#!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then exit 0 fi # Defined as %py_reproducible_pyc_path macro and passed here as # the first command-line argument -path_to_fix=$1 +path_to_fix=${1:?} # First, check that the parser is available: if [ ! -x /usr/bin/marshalparser ]; then @@ -15,6 +15,4 @@ if [ ! -x /usr/bin/marshalparser ]; then exit 1 fi -# Set pipefail so if $path_to_fix does not exist, the build fails -set -o pipefail -find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite +find "$path_to_fix" -type f -name '*.pyc' -exec /usr/bin/marshalparser --fix --overwrite '{}' '+' diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3c2511..5d2daf2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 +- Minor improvements to brp-fix-pyc-reproducibility + * Fri Mar 22 2024 Lumír Balhar - 3.12-8 - Update bundled compileall2 to version 0.8.0 From a389cc467425766a0122be2c143a9a4fdb76191e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 16 Oct 2023 16:45:02 +0200 Subject: [PATCH 199/218] Update main Python to 3.13 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 278571f..d57e43f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.12 +%__default_python3_version 3.13 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 5d2daf2..ffd4d9a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Jun 06 2024 Karolina Surma - 3.13-1 +- Update main Python to 3.13 + * Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 - Minor improvements to brp-fix-pyc-reproducibility From d430201e60a2d911f58bb979b1eca6299bbc96c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 25 Jun 2024 12:34:11 +0200 Subject: [PATCH 200/218] CI tests fix for Python 3.13 We should not unimport pathlib when we use pathlib. This was always fragile. Choose a module we don't use: The failure: ______________________ test_modules_from_files_are_found _______________________ tmp_path = PosixPath('/tmp/pytest-of-root/pytest-0/test_modules_from_files_are_fo0') def test_modules_from_files_are_found(tmp_path): test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') test_file_2.write_text('csv\npathlib\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): sys.modules.pop(m, None) > modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) test_import_all_modules.py:170: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/rpm/redhat/import_all_modules.py:167: in main import_modules(modules) /usr/lib/rpm/redhat/import_all_modules.py:100: in import_modules importlib.import_module(module) /usr/lib64/python3.13/importlib/__init__.py:88: in import_module return _bootstrap._gcd_import(name[level:], package, level) :1387: in _gcd_import ??? :1360: in _find_and_load ??? :1331: in _find_and_load_unlocked ??? :935: in _load_unlocked ??? :1022: in exec_module ??? :488: in _call_with_frames_removed ??? _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ """Object-oriented filesystem paths. This module provides classes to represent abstract paths and concrete paths with operations that have semantics appropriate for different operating systems. """ from ._abc import * from ._local import * > __all__ = (_abc.__all__ + _local.__all__) E NameError: name '_abc' is not defined. Did you forget to import '_abc'? /usr/lib64/python3.13/pathlib/__init__.py:11: NameError ----------------------------- Captured stderr call ----------------------------- Check import: math Check import: wave Check import: csv Check import: pathlib --- tests/test_import_all_modules.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 71feeff..8b89bc4 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -160,15 +160,15 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('csv\npathlib\n') + test_file_2.write_text('csv\nnetrc\n') test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'csv', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'netrc', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('csv', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'netrc', 'logging'): assert module in sys.modules From 840a26c515db6b45fda5bce35422db3a7c2f09d5 Mon Sep 17 00:00:00 2001 From: Cristian Le Date: Wed, 10 Apr 2024 09:33:39 +0200 Subject: [PATCH 201/218] Add option -a to include BuilArch: noarch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Option -A disables that (the default, does nothing at the moment) Co-Authored-By: Miro Hrončok --- macros.python-srpm | 15 +++++++++++++-- python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index d57e43f..00098d2 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -225,15 +225,19 @@ end } -%python_extras_subpkg(n:i:f:F) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' local option_F = '-F (skip %%files section)' + local option_a = '-a (insert BuildArch: noarch)' + local option_A = '-A (do not insert BuildArch: noarch (default))' local value_n = rpm.expand('%{-n*}') local value_i = rpm.expand('%{-i*}') local value_f = rpm.expand('%{-f*}') local value_F = rpm.expand('%{-F}') + local value_a = rpm.expand('%{-a}') + local value_A = rpm.expand('%{-A}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -250,6 +254,9 @@ if value_f ~= '' and value_F ~= '' then rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') end + if value_a ~= '' and value_A ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_a .. ' and ' .. option_A .. ' options are not possible}') + end if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end @@ -277,7 +284,11 @@ elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end - for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do + local tags = summary .. '\\\n' .. requires + if value_a ~= '' then + tags = tags .. '\\\nBuildArch: noarch' + end + for i, line in ipairs({pkgdef, tags, description, files, ''}) do print(line .. '\\\n') end end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ffd4d9a..7d3d1ef 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Jun 25 2024 Cristian Le - 3.13-2 +- %%python_extras_subpkg: Add option -a to include BuildArch: noarch + * Thu Jun 06 2024 Karolina Surma - 3.13-1 - Update main Python to 3.13 diff --git a/tests/test_evals.py b/tests/test_evals.py index 1f953e0..e9fa44a 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -619,6 +619,45 @@ def test_python_extras_subpkg_F(): assert lines == expected +def test_python_extras_subpkg_a(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + BuildArch: noarch + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_A(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_aA(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -A -F toml', + version='6', release='7', fails=True) + assert lines[0] == ('error: %python_extras_subpkg: simultaneous -a ' + '(insert BuildArch: noarch) and -A (do not insert ' + 'BuildArch: noarch (default)) options are not possible') + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From d9e31f78970f00cbb27136960c602845ace55ea2 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 19 Jul 2024 15:32:23 +0000 Subject: [PATCH 202/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 7d3d1ef..a2e5ca0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + * Tue Jun 25 2024 Cristian Le - 3.13-2 - %%python_extras_subpkg: Add option -a to include BuildArch: noarch From ef265656475b091f40dae1d8c229a03257f25bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 17:19:32 +0200 Subject: [PATCH 203/218] Stop testing Python 2.7 on the CI, we don't have it any more --- tests/pythontest.spec | 13 +++---------- tests/tests.yml | 1 - 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 32bb039..26eaa2c 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,9 +1,8 @@ %global basedir /opt/test/byte_compilation # We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 -# Test with a representative of each. +# Test with a representative of each, except 2.7 which we no longer have %global python36_sitelib /usr/lib/python3.6/site-packages -%global python27_sitelib /usr/lib/python2.7/site-packages Name: pythontest Version: 0 @@ -12,7 +11,6 @@ Summary: ... License: MIT BuildRequires: python3-devel BuildRequires: python3.6 -BuildRequires: python2.7 %description ... @@ -33,23 +31,19 @@ echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py mkdir -p %{buildroot}%{python36_sitelib}/directory/ echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py -mkdir -p %{buildroot}%{python27_sitelib}/directory/ -echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py - %check LOCATIONS=" %{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/ %{buildroot}%{python36_sitelib}/directory/ - %{buildroot}%{python27_sitelib}/directory/ " # Count .py and .pyc files PY=$(find $LOCATIONS -name "*.py" | wc -l) PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) -# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7) -test $PY -eq 5 +# We should have 4 .py files (3 for python3, one for 3.6) +test $PY -eq 4 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files @@ -70,7 +64,6 @@ test $PY -ge $INODES %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py -%{python27_sitelib}/directory/file.py* %changelog diff --git a/tests/tests.yml b/tests/tests.yml index 4b8b2dd..af2c2a1 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -36,5 +36,4 @@ - python3-devel - python3-pytest - python3.6 - - python2.7 From 2c230bfcac1e97875f47d01e40364a55f44038e1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Sat, 18 Jan 2025 19:00:07 +0000 Subject: [PATCH 204/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a2e5ca0..bcae6d1 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -53,7 +53,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -163,6 +163,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild + * Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 - Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild From a3ba11ea648fda3b08613176a448b93a53386d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Hrn=C4=8Diar?= Date: Fri, 14 Feb 2025 10:14:05 +0100 Subject: [PATCH 205/218] Add BuildRoot Policy script to modify the content of .dist-info/INSTALLER file Fixes: rhbz#2345186 --- brp-python-rpm-in-distinfo | 15 +++++++++++++ macros.python-srpm | 11 +++++++--- python-rpm-macros.spec | 9 +++++++- tests/test_rpm_in_distinfo.py | 41 +++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) create mode 100755 brp-python-rpm-in-distinfo create mode 100644 tests/test_rpm_in_distinfo.py diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo new file mode 100755 index 0000000..473af00 --- /dev/null +++ b/brp-python-rpm-in-distinfo @@ -0,0 +1,15 @@ +#!/bin/sh + +set -eu +# If using normal root, avoid changing anything. +if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then + exit 0 +fi + +find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do + if cmp -s <(echo pip) "$installer" ; then + echo "rpm" > "$installer" + rm -f $(dirname "$installer")/RECORD + fi +done +exit 0 diff --git a/macros.python-srpm b/macros.python-srpm index 00098d2..f26ff0c 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -66,6 +66,8 @@ ### BRP scripts (and related macros) +## Modifies installation method in .dist-info/INSTALLER file to rpm +%python_rpm_in_distinfo 1 ## Automatically compile python files %py_auto_byte_compile 1 ## Should python bytecompilation errors terminate a build? @@ -78,16 +80,19 @@ %__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts +%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink ## This macro is included in redhat-rpm-config's %%__os_install_post # Note that the order matters: -# 1. brp-python-bytecompile can create (or replace) pyc files -# 2. brp-fix-pyc-reproducibility can modify the pyc files from above -# 3. brp-python-hardlink de-duplicates identical pyc files +# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file +# 2. brp-python-bytecompile can create (or replace) pyc files +# 3. brp-fix-pyc-reproducibility can modify the pyc files from above +# 4. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ + %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index bcae6d1..189c94a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -33,6 +33,8 @@ Source402: brp-python-hardlink # This one is from redhat-rpm-config < 190 # It has no upstream yet Source403: brp-fix-pyc-reproducibility +# brp script to write "rpm" string into the .dist-info/INSTALLER file +Source404: brp-python-rpm-in-distinfo # macros and lua: MIT # import_all_modules.py: MIT @@ -53,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -136,6 +138,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} %global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} %global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} +%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} %check @@ -156,6 +159,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -163,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 +- Add brp script to modify .dist-info/INSTALLER file + * Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 - Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py new file mode 100644 index 0000000..7546dc0 --- /dev/null +++ b/tests/test_rpm_in_distinfo.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import os +import pytest +import subprocess + +@pytest.fixture +def create_test_files(tmp_path): + def _create(subpath, installer_content): + dir_path = tmp_path / subpath + dir_path.mkdir(parents=True, exist_ok=True) + installer_file = dir_path / "INSTALLER" + installer_file.write_text(installer_content) + record_file = dir_path / "RECORD" + record_file.write_text("dummy content in RECORD file\n") + return dir_path + return _create + +testdata = [ + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), +] +@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) +def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, +path, installer_content, expected_installer_content, record_file_exists): + script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") + tmp_dir = create_test_files(path, installer_content) + monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) + result = subprocess.run( + [script_path], + capture_output=True, text=True + ) + + assert result.returncode == 0 + assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content + assert Path(tmp_dir / "RECORD").exists() is record_file_exists + From 4ddd2ea298c11703dd97252462270cadc6a438c7 Mon Sep 17 00:00:00 2001 From: Pavlina Moravcova Varekova Date: Fri, 9 Aug 2019 16:30:43 +0200 Subject: [PATCH 206/218] Eliminate use of ambiguous logical operators in script conditionals Prefer '[] && []' to '[ -a ]' and '[] || []' to '[ -o ]' in tests. -a and -o to mean AND and OR in a [ .. ] test expression is not well defined, and can cause incorrect results when arguments start with dashes or contain !. Moreover binary -a and -o are inherently ambiguous. test(1) man page recommends to use 'test EXPR1 && test EXPR2' or 'test EXPR1 || test EXPR2' instead. It corrects warnings [SC2166] spotted by covscan. --- brp-python-bytecompile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 1f7c4cd..b1e5db5 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then + if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi From 888775f1c5505bded0e8e8f2baf93b145454840e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 28 May 2025 10:32:43 +0200 Subject: [PATCH 207/218] Switch default Python version to 3.14 --- macros.python-srpm | 2 +- python-rpm-macros.spec | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f26ff0c..7ad62d1 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.13 +%__default_python3_version 3.14 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 189c94a..9faa80e 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 1%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed May 28 2025 Karolina Surma - 3.14-1 +- Update main Python to 3.14 + * Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 - Add brp script to modify .dist-info/INSTALLER file From a74d2bb5c9a69b7b90b8afc5560f8ed4e95b2232 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 18:35:59 -0700 Subject: [PATCH 208/218] Minor style fixes suggested by ShellCheck. Mostly, these consist of preferring '[[' to '[' in bash scripts. Other changes include quoting unquoted variables, and explicitly specifying bash as the interpreter for scripts that use features not defined in POSIX sh Fixes SC2046, SC3001, and SC2292 --- brp-fix-pyc-reproducibility | 4 ++-- brp-python-bytecompile | 20 ++++++++++---------- brp-python-rpm-in-distinfo | 6 +++--- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 16613b0..05857b3 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,7 +1,7 @@ #!/bin/bash -eu # If using normal root, avoid changing anything. -if [ -z "${RPM_BUILD_ROOT:-}" ] || [ "${RPM_BUILD_ROOT:-}" = "/" ]; then +if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then exit 0 fi @@ -10,7 +10,7 @@ fi path_to_fix=${1:?} # First, check that the parser is available: -if [ ! -x /usr/bin/marshalparser ]; then +if [[ ! -x /usr/bin/marshalparser ]]; then echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" exit 1 fi diff --git a/brp-python-bytecompile b/brp-python-bytecompile index b1e5db5..fd7172f 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -6,7 +6,7 @@ errors_terminate=$2 # Therefore $1 ($default_python) is not needed and is invoked with "" by default. # $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 -if [ 0$extra -eq 1 ]; then +if [[ 0"$extra" -eq 1 ]]; then echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr exit 1 fi @@ -14,7 +14,7 @@ fi compileall_flags="$4" # If using normal root, avoid changing anything. -if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then +if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then exit 0 fi @@ -45,14 +45,14 @@ function python_bytecompile() # # Python 3.4 and higher # - if [ "$python_version" -ge 34 ]; then + if [[ "$python_version" -ge 34 ]]; then # We compile all opt levels in one go: only when $options is empty. - if [ -n "$options" ]; then + if [[ -n "$options" ]]; then return fi - if [ "$python_version" -ge 39 ]; then + if [[ "$python_version" -ge 39 ]]; then # For Pyhon 3.9+, use the standard library compileall_module=compileall else @@ -60,7 +60,7 @@ function python_bytecompile() compileall_module=compileall2 fi - if [ "$python_version" -ge 37 ]; then + if [[ "$python_version" -ge 37 ]]; then # Force the TIMESTAMP invalidation mode invalidation_option=--invalidation-mode=timestamp else @@ -69,7 +69,7 @@ function python_bytecompile() invalidation_option= fi - [ ! -z $exclude ] && exclude="-x '$exclude'" + [[ -n "$exclude" ]] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels @@ -137,12 +137,12 @@ do # Generate normal (.pyc) byte-compiled files. python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had inaccessible mtime exit 1 fi python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi @@ -150,7 +150,7 @@ do # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [ $? -ne 0 ] && [ 0$errors_terminate -ne 0 ]; then + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then # One or more of the files had a syntax error exit 1 fi diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo index 473af00..b72d704 100755 --- a/brp-python-rpm-in-distinfo +++ b/brp-python-rpm-in-distinfo @@ -1,15 +1,15 @@ -#!/bin/sh +#!/usr/bin/bash set -eu # If using normal root, avoid changing anything. -if [ "${RPM_BUILD_ROOT:-/}" = "/" ] ; then +if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then exit 0 fi find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do if cmp -s <(echo pip) "$installer" ; then echo "rpm" > "$installer" - rm -f $(dirname "$installer")/RECORD + rm -f "$(dirname "$installer")/RECORD" fi done exit 0 From 6b1cf3771c4370e61592d076fb8d313a5cff2519 Mon Sep 17 00:00:00 2001 From: Gordon Messmer Date: Sun, 18 May 2025 22:35:06 -0700 Subject: [PATCH 209/218] The "exclude" variable in python_bytecompile is no longer used, so remove it. This resolves ShellCheck SC2089 and SC2090 warnings. --- brp-python-bytecompile | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/brp-python-bytecompile b/brp-python-bytecompile index fd7172f..1911fa1 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -36,7 +36,7 @@ function python_bytecompile() { local options=$1 local python_binary=$2 - local exclude=$3 + # local exclude=$3 # No longer used local python_libdir="$4" local compileall_flags="$5" @@ -69,18 +69,14 @@ function python_bytecompile() invalidation_option= fi - [[ -n "$exclude" ]] && exclude="-x '$exclude'" - # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels # -q disables verbose output # -f forces the process to overwrite existing compiled files - # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root - # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -96,13 +92,10 @@ python_libdir = "$python_libdir" depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" -exclude = r"$exclude" class Filter: def search(self, path): ret = not os.path.realpath(path).startswith(build_root) - if exclude: - ret = ret or re.search(exclude, path) return ret sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) From b8a5807572e25276e83fbdc2f2a2d26f7e4592a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 23 Sep 2024 16:55:23 +0200 Subject: [PATCH 210/218] Deprecate %py3_build, %py3_build_wheel, and %py3_install ...as well as their %py_... counterparts. https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros --- macros.python | 23 ++++++++++++++++++++--- macros.python3 | 6 +++--- python-rpm-macros.spec | 7 ++++++- tests/test_evals.py | 34 +++++++++++++++++++++++++++++++++- 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/macros.python b/macros.python index 0ba4d78..4bd0da6 100644 --- a/macros.python +++ b/macros.python @@ -23,6 +23,23 @@ end print(_python_macro_cache[cache_key][name]) } +# Deprecation wrapper, warns only once per macro +# Options: +# -n - The name of the macro that is deprecated +%_python_deprecated(n:) %{lua: +if not _python_deprecated_warned then + -- This is intentionally a global lua table + _python_deprecated_warned = {} +end +if not _python_deprecated_warned[opt.n] then + _python_deprecated_warned[opt.n] = true + local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. + "See the current Python packaging guidelines: " .. + "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" + macros.warn({msg}) +end +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) @@ -69,17 +86,17 @@ print(_python_macro_cache[cache_key][name]) # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{expand:\\\ +%py_build() %{_python_deprecated -n py_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_wheel() %{expand:\\\ +%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{expand:\\\ +%py_install() %{_python_deprecated -n py_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/macros.python3 b/macros.python3 index 7197433..38cdcd4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -43,17 +43,17 @@ # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{expand:\\\ +%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_wheel() %{expand:\\\ +%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{expand:\\\ +%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 9faa80e..af7188c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 1%{?dist} +Release: 2%{?dist} BuildArch: noarch @@ -167,6 +167,11 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Sun Jun 29 2025 Miro Hrončok - 3.14-2 +- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install +- Deprecate %%py_build, %%py_build_wheel, and %%py_install +- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros + * Wed May 28 2025 Karolina Surma - 3.14-1 - Update main Python to 3.14 diff --git a/tests/test_evals.py b/tests/test_evals.py index e9fa44a..b26ff29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -760,6 +760,9 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) + # strip the deprecation message + if 'deprecated' in lines[0]: + lines = lines[1:] assert lines[0] == ( 'error: attempt to use unversioned python, ' 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' @@ -777,7 +780,9 @@ def test_unversioned_python_errors(macro): @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + unverisoned = rpm_eval(macro, __python='/usr/bin/python3') + expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] + assert unverisoned == expected # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -958,3 +963,30 @@ def test_multi_python3(alt_x_y): lines = rpm_eval(evals) lines = [l for l in lines if l] # strip empty lines generated by %global assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +@pytest.mark.parametrize('macro', [ + '%py3_build', + '%py3_build_wheel', + '%py3_install', +]) +def test_deprecation(macro): + lines = rpm_eval(macro) + assert "is deprecated" in lines[0] + assert f"{macro} " in lines[0] + + +def test_multiple_deprecation(): + source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' + lines = rpm_eval(source) + + assert "is deprecated" in lines[0] + assert "%py3_build " in lines[0] + + assert "is deprecated" in lines[1] + assert "%py3_build_wheel " in lines[1] + + assert "is deprecated" in lines[2] + assert "%py3_install " in lines[2] + + assert "is deprecated" not in '\n'.join(lines[3:]) From 5cdc4d85a735c77daaa339f128f69ef98e9b0171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8D=C3=B1igo=20Huguet?= Date: Mon, 21 Jul 2025 08:07:36 +0200 Subject: [PATCH 211/218] pathfix.py: Don't fail on symbolic links The script ignores symlinks for obvious reasons. Don't return an error code in that case, though, as it makes the rpm builds fail if there are symlinks in /usr/bin and `pathfix.py ... /usr/bin/*` is used (e.g. via %pyproject_install from pyproject-rpm-macros). --- pathfix.py | 1 - python-rpm-macros.spec | 5 ++++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pathfix.py b/pathfix.py index 1d7db3a..6808382 100644 --- a/pathfix.py +++ b/pathfix.py @@ -56,7 +56,6 @@ def main(): if recursedown(arg): bad = 1 elif os.path.islink(arg): err(arg + ': will not process symbolic links\n') - bad = 1 else: if fix(arg): bad = 1 sys.exit(bad) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index af7188c..859d687 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 2%{?dist} +Release: 3%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 +- pathfix.py: Don't fail on symbolic links + * Sun Jun 29 2025 Miro Hrončok - 3.14-2 - Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install - Deprecate %%py_build, %%py_build_wheel, and %%py_install From 066459f836121418875c01b7f8bbc59980fe7be1 Mon Sep 17 00:00:00 2001 From: Fedora Release Engineering Date: Fri, 25 Jul 2025 10:14:58 +0000 Subject: [PATCH 212/218] Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild --- python-rpm-macros.spec | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 859d687..a3a8be5 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 3%{?dist} +Release: 4%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Jul 25 2025 Fedora Release Engineering +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + * Mon Jul 21 2025 Íñigo Huguet - 3.14-3 - pathfix.py: Don't fail on symbolic links From bfd1bc9738592449f8ddd6308c3d1b32c10243aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Luk=C3=A1=C5=A1=20Zachar?= Date: Fri, 25 Jul 2025 18:19:58 +0200 Subject: [PATCH 213/218] Drop STI and use tmt instead Resolves: rhbz#2383044 --- .fmf/version | 1 + plan.fmf | 35 +++++++++++++++++++++++++++++++++++ tests/tests.yml | 39 --------------------------------------- 3 files changed, 36 insertions(+), 39 deletions(-) create mode 100644 .fmf/version create mode 100644 plan.fmf delete mode 100644 tests/tests.yml diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/plan.fmf b/plan.fmf new file mode 100644 index 0000000..a681cdf --- /dev/null +++ b/plan.fmf @@ -0,0 +1,35 @@ +execute: + how: tmt + +discover: + - name: same_repo + how: shell + tests: + - name: pytest + test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - name: manual_byte_compilation_clamp_mtime_off + path: /tests + test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - name: manual_byte_compilation_clamp_mtime_on + path: /tests + test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - name: rpmlint_clamp_mtime_off + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: rpmlint_clamp_mtime_on + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + +prepare: + - name: Install dependencies + how: install + package: + - rpm-build + - rpmlint + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-pytest + - python3.6 + - dnf + - name: Update packages + how: shell + script: dnf upgrade -y diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index af2c2a1..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -- hosts: localhost - tags: - - classic - tasks: - - dnf: - name: "*" - state: latest - -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - tests: - - pytest: - dir: . - run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - manual_byte_compilation_clamp_mtime_off: - dir: . - run: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - - manual_byte_compilation_clamp_mtime_on: - dir: . - run: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec - - rpmlint_clamp_mtime_off: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - rpmlint_clamp_mtime_on: - dir: . - run: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - required_packages: - - rpm-build - - rpmlint - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-pytest - - python3.6 - From 364d99f4e1bd27fedeec5a1e8a7008f5bf5b5302 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 5 Aug 2025 20:16:56 +0200 Subject: [PATCH 214/218] import_all_modules: Add error handling for import failures Continue checking all modules when imports fail and provide detailed error reporting. Exit with proper status code when failures occur. --- import_all_modules.py | 21 ++++++++++- python-rpm-macros.spec | 5 ++- tests/test_import_all_modules.py | 64 +++++++++++++++++++++++++++----- 3 files changed, 78 insertions(+), 12 deletions(-) diff --git a/import_all_modules.py b/import_all_modules.py index 3930236..97e924c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,6 +7,7 @@ import os import re import site import sys +import traceback from contextlib import contextmanager from pathlib import Path @@ -93,11 +94,24 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. + + Return a list of failed modules. ''' + failed_modules = [] + for module in modules: print('Check import:', module, file=sys.stderr) - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception: + traceback.print_exc(file=sys.stderr) + failed_modules.append(module) + + if failed_modules: + print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) + + return failed_modules def argparser(): @@ -164,7 +178,10 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - import_modules(modules) + failed_modules = import_modules(modules) + + if failed_modules: + raise SystemExit(1) if __name__ == '__main__': diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a3a8be5..d536935 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -55,7 +55,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 4%{?dist} +Release: 5%{?dist} BuildArch: noarch @@ -167,6 +167,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Mon Aug 11 2025 Lumír Balhar - 3.14-5 +- import_all_modules: Add error handling for import failures + * Fri Jul 25 2025 Fedora Release Engineering - Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 8b89bc4..dd10689 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['this_is_a_module_in_cwd']) @@ -175,7 +175,7 @@ def test_modules_from_files_are_found(tmp_path): def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,24 +253,70 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_module_with_error_propagates_exception(tmp_path): +def test_import_module_returns_failed_modules(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + assert failed_modules == ['this_is_a_module_in_tmp_path_2'] + + +def test_module_with_error_propagates_exception(tmp_path, capsys): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - # The correct exception must be raised - with pytest.raises(ZeroDivisionError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) + _, err = capsys.readouterr() + assert "ZeroDivisionError" in err + + +def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) + assert failed_modules == [] + + +def test_all_modules_are_imported(tmp_path, capsys): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + with pytest.raises(SystemExit): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'missing_module', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ]) + _, err = capsys.readouterr() + for i in range(1, 4): + assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err + assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): From 7d4cb5437d4f813b20dc2be44fe5d6517f48464f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 22 Jul 2025 20:51:50 +0200 Subject: [PATCH 215/218] Introduce %python_wheel_inject_sbom See https://discuss.python.org/t/encoding-origin-in-wheel-and-dist-info-metadata-for-downstream-security-backports/97436 --- macros.python-wheel-sbom | 124 +++++++++++++++++++++++++++++++++++++++ plan.fmf | 5 ++ python-rpm-macros.spec | 7 ++- tests/testwheel.spec | 102 ++++++++++++++++++++++++++++++++ 4 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 macros.python-wheel-sbom create mode 100644 tests/testwheel.spec diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom new file mode 100644 index 0000000..a21460a --- /dev/null +++ b/macros.python-wheel-sbom @@ -0,0 +1,124 @@ +# The macros in this file are used to add SBOM to wheel files that we ship. +# Majority of Python packages will not need to do that, +# as they only use wheels as an intermediate artifact. +# The macros will be used by packages installing wheel to %%python_wheel_dir +# or by Python interpreters bundling their own (patched) wheels. +# +# The runtime dependencies are not Required by the python-rpm-macros package, +# users of this macro need to specify them on their own or rely on the fact that +# they are all available in the default buildroot. +# +# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS +# +# The wheels are modified in-place. + + +# Path of the SBOM file in the PEP 770 .dist-info/sboms directory +# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ +# section Recognized file patterns +%__python_wheel_sbom_filename bom.json + + +# The SBOM content to put to the file +# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 +%__python_wheel_sbom_content %{expand:{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "%{name}", + "version": "%{version}-%{release}", + "purl": "%{__python_wheel_purl}" + } + ] +}} + + +# The purl used above +# We use the src package name (which is easier to get and more useful to consumers). +# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 +# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ +%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src + + +# The purl namespace used above +# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ +# Distributors, define %%dist_purl_namespace to set this. +# The rest of the code is fallback for distributions without it (relying on %%dist_name). +%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: + if macros.epel then + -- being epel beats the %%dist_name value + -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 + print("epel") + else + local dist_map = { + -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm + -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 + ["Fedora Linux"] = "fedora", + -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 + ["CentOS Stream"] = "centos", + -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ + ["Red Hat Enterprise Linux"] = "redhat", + -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html + ["AlmaLinux"] = "almalinux", + -- from https://github.com/google/osv.dev/pull/2939 + ["Rocky Linux"] = "rocky-linux", + } + print(dist_map[macros.dist_name] or "unknown") + end +}} + + +# A Bash scriptlet to inject the SBOM file into the wheel(s) +# The macro takes positional nargs+ with wheel paths +# For each wheel, it +# 1. aborts if the SBOM file is already there (it won't override) +# 2. inserts the SBOM file to .dist-info/sboms +# 3. amends .dist-info/RECORD with the added SBOM file +%python_wheel_inject_sbom() %{expand:( + %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] + + set -eu -o pipefail + export LANG=C.utf-8 + + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + pwd0=$(pwd) + ret=0 + + for whl in %{*}; do + cd "$tmpdir" + if [[ "$whl" != /* ]]; then + whl="$pwd0/$whl" + fi + + record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + distinfo="${record%%/RECORD}" + bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" + + if zipinfo -1 "$whl" | grep -qFx "$bom"; then + echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 + ret=1 + continue + fi + + unzip "$whl" "$record" + mkdir "$distinfo/sboms" + echo '%{__python_wheel_sbom_content}' > "$bom" + checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" + size="$(wc --bytes "$bom" | cut -f1 -d' ')" + echo "$bom,$checksum,$size" >> "$record" + + if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then + touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" + fi + + zip -r "$whl" "$record" "$bom" + rm -rf "$distinfo" + cd "$pwd0" + done + + exit $ret +)} + diff --git a/plan.fmf b/plan.fmf index a681cdf..88850dd 100644 --- a/plan.fmf +++ b/plan.fmf @@ -17,6 +17,9 @@ discover: test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - name: rpmlint_clamp_mtime_on test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: python_wheel_inject_sbom + path: /tests + test: rpmbuild -ba testwheel.spec prepare: - name: Install dependencies @@ -27,6 +30,8 @@ prepare: - python-rpm-macros - python3-rpm-macros - python3-devel + - python3-setuptools + - python3-pip - python3-pytest - python3.6 - dnf diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index d536935..86d2865 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -8,6 +8,7 @@ Source101: macros.python Source102: macros.python-srpm Source104: macros.python3 Source105: macros.pybytecompile +Source106: macros.python-wheel-sbom # Lua files Source201: python.lua @@ -55,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 5%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -149,6 +150,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/pathfix.py @@ -167,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Wed Aug 13 2025 Miro Hrončok - 3.14-6 +- Introduce %%python_wheel_inject_sbom + * Mon Aug 11 2025 Lumír Balhar - 3.14-5 - import_all_modules: Add error handling for import failures diff --git a/tests/testwheel.spec b/tests/testwheel.spec new file mode 100644 index 0000000..2e761fc --- /dev/null +++ b/tests/testwheel.spec @@ -0,0 +1,102 @@ +Name: testwheel +Epoch: 42 +Version: 1 +Release: 0%{?dist} +Summary: ... +License: MIT +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools >= 61 +BuildRequires: python3-pip + +%description +This builds and installs a wheel which we can then use as a test for +%%python_wheel_inject_sbom. + + +%prep +cat > pyproject.toml << EOF +[project] +name = "testwheel" +version = "1" + +[build-system] +requires = ["setuptools >= 61"] +build-backend = "setuptools.build_meta" +EOF + + +%build +export PIP_CONFIG_FILE=/dev/null +%{python3} -m pip wheel . --no-build-isolation + +# The macro should happily alter multiple wheels, let's make more +for i in {1..5}; do + mkdir ${i} + cp -a *.whl ${i} +done + +# using relative paths should succeed +%python_wheel_inject_sbom {1..5}/*.whl + +# repetitive use should bail out and fail (SBOM is already there) +%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true + +# each wheel should already have it, all should fail individually as well +for i in {1..5}; do + %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true +done + + +%install +mkdir -p %{buildroot}%{python_wheel_dir} +cp -a *.whl %{buildroot}%{python_wheel_dir} + +# using absolute paths should work +%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl + +# and fail when repeated +%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true + + +%check +%define venvsite venv/lib/python%{python3_version}/site-packages +%{python3} -m venv venv +venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl + +test -f %{venvsite}/testwheel-1.dist-info/RECORD +test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD +# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) +sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' + +# this deliberately uses a different mechanism than the macro +# if you are running this test on a different distro, adjust it +%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} + +PYTHONOPTIMIZE=0 %{python3} -c " +import json +with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: + sbom = json.load(fp) +assert len(sbom['components']) == 1 +assert sbom['components'][0]['type'] == 'library' +assert sbom['components'][0]['name'] == 'testwheel' +assert sbom['components'][0]['version'] == '1-0%{?dist}' +assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' +" + +# replace the installation with the original unaltered wheel +venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl +test -f %{venvsite}/testwheel-1.dist-info/RECORD +# no SBOM +test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true + + +%files +%{python_wheel_dir}/*.whl + + +%changelog +* Wed Aug 13 2025 Miro Hrončok - 42:1-0 +- A static changelog with a date, so we can clamp mtimes From 47de23b3c094fafcace77131f4ce6c233f5d1c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 29 Aug 2025 13:46:23 +0200 Subject: [PATCH 216/218] %python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos In python-setuptools-wheel, the macro was confused by setuptools/_vendor/autocommand-2.2.2.dist-info/RECORD (or other vendored RECORDs) --- macros.python-wheel-sbom | 2 +- python-rpm-macros.spec | 5 ++++- tests/testwheel.spec | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom index a21460a..41389f9 100644 --- a/macros.python-wheel-sbom +++ b/macros.python-wheel-sbom @@ -93,7 +93,7 @@ whl="$pwd0/$whl" fi - record=$(zipinfo -1 "$whl" | grep '\.dist-info/RECORD$') + record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') distinfo="${record%%/RECORD}" bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 86d2865..ae3d558 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 6%{?dist} +Release: 7%{?dist} BuildArch: noarch @@ -169,6 +169,9 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Fri Aug 29 2025 Miro Hrončok - 3.14-7 +- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos + * Wed Aug 13 2025 Miro Hrončok - 3.14-6 - Introduce %%python_wheel_inject_sbom diff --git a/tests/testwheel.spec b/tests/testwheel.spec index 2e761fc..461d7f3 100644 --- a/tests/testwheel.spec +++ b/tests/testwheel.spec @@ -23,7 +23,18 @@ version = "1" [build-system] requires = ["setuptools >= 61"] build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["testwheel*"] EOF +# create a secondary dist-info folder in the project +# we need to ensure this file is not altered +mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info +touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +echo 'recursive-include testwheel/_vendor *' > MANIFEST.in %build @@ -70,6 +81,9 @@ grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/ # a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true + # this deliberately uses a different mechanism than the macro # if you are running this test on a different distro, adjust it %define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} From 9e5c1461f2346d245a18e3650834d548647dff48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 9 Sep 2025 15:00:10 +0200 Subject: [PATCH 217/218] %python_extras_subpkg: Add -v option to specify the required version(-release) This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package). --- macros.python-srpm | 6 ++++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index 7ad62d1..f575868 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -230,7 +230,7 @@ end } -%python_extras_subpkg(n:i:f:FaA) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' @@ -243,6 +243,7 @@ local value_F = rpm.expand('%{-F}') local value_a = rpm.expand('%{-a}') local value_A = rpm.expand('%{-A}') + local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -265,7 +266,8 @@ if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index ae3d558..032c913 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Tue Sep 09 2025 Miro Hrončok - 3.14-8 +- %%python_extras_subpkg: Add -v option to specify the required version(-release) +- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) + * Fri Aug 29 2025 Miro Hrončok - 3.14-7 - %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos diff --git a/tests/test_evals.py b/tests/test_evals.py index b26ff29..31fb75d 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -658,6 +658,21 @@ def test_python_extras_subpkg_aA(): 'BuildArch: noarch (default)) options are not possible') +def test_python_extras_subpkg_v(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 1.2.3 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') From 0d417402dbc243c6fa3170fff3e8d6b5b59638c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 16 Oct 2025 16:10:53 +0200 Subject: [PATCH 218/218] %python_extras_subpkg: Only %ghost the egg-info/dist-info directory, not the content That way, accidentally unpackaged files within are reported as errors. Currently, when %python_extras_subpkg is used, the egg-info/dist-info directory is packaged as %ghost. When the main package does not have it, the RPM build would succeed. The extras packages would have the python3dist() requires and provides, but the main package would not. By adding %dir after %ghost, we only package the directory (which is enough for python3-rpm-generators to process it), but the files in the directory are not included. When not packaged in the main package, the RPM build fails. This is a safeguard against packaging mistakes. The visible difference is that rpm -ql/repoquery -l would only return the metadata directory. And the RPM build would fail if .egg-info is a file, which is only possible with Python < 3.12 for packages using distutils (no extras anyway). --- macros.python-srpm | 2 +- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index f575868..d5cd4e5 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -287,7 +287,7 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 032c913..263853a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -56,7 +56,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} BuildArch: noarch @@ -169,6 +169,10 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog +* Thu Oct 16 2025 Miro Hrončok - 3.14-9 +- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content +- That way, accidentally unpackaged files within are reported as errors + * Tue Sep 09 2025 Miro Hrončok - 3.14-8 - %%python_extras_subpkg: Add -v option to specify the required version(-release) - This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) diff --git a/tests/test_evals.py b/tests/test_evals.py index 31fb75d..e6cc8f7 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -551,7 +551,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -562,7 +562,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected