Architecture-specific pyc files cause two problems: - noarch packages may cause noarch packages to differ between different architectures in the case where we build an archful package which has some noarch subpackages, - package rebuilds report a difference when build reproducibility is checked and the build happened on a different architecture. Both problems can be resolved by using the existing %py_reproducible_pyc_path macro, but this requires maintainers to opt-in. Let's just extend this operation to all files under /usr, to make life easier for maintainers and for folks working on reproducible builds. "find /usr/ -name '*.pyc'" reveals that .pyc files are installed in quite a few locations, not just the python site and arch directories, so just apply the fixer to the whole /usr. %py_fix_reproducibility can be unset to opt out. %py_reproducible_pyc_path retains its function. We can probably drop all or almost all of its uses in Fedora, but it might be useful for people doing some special things. Fixes https://pagure.io/fedora-reproducible-builds/project/issue/12, https://bugzilla.redhat.com/show_bug.cgi?id=2266767. To deal with bootstrapping, the script does nothing if marshalparser is not installed. python-srpm-macros Require marshalparser, but not when built with bootstrap. The call to marshalparser takes a bit of time. I ran it on all .pyc files on my system and that took a few minutes. But for each single package, the time is usually < 1 s. There are some exceptions, for example libvirt.cpython-312.*.pyc take ~15 s each. But I don't think this really matters: the files come from a huge package which takes a long time to compile, and adding a few dozen seconds at the end doesn't change much. I expect that marshalparser itself might need some optimizations. We can do that at some later point if the slowdowns become noticeable.
288 lines
12 KiB
Text
288 lines
12 KiB
Text
# There are multiple Python 3 versions packaged, but only one can be the "main" version
|
|
# That means that it owns the "python3" namespace:
|
|
# - python3 package name
|
|
# - /usr/bin/python3 command
|
|
# - python3-foo packages are meant for this version
|
|
# Other versions of Python 3 always contain the version in the namespace:
|
|
# - python3.XX package name
|
|
# - /usr/bin/python3.XX command
|
|
# - python3.XX-foo packages (if allowed)
|
|
#
|
|
# Python spec files use the version defined here to determine defaults for the
|
|
# %%py_provides and %%python_provide macros, as well as for the "pythonname" generator that
|
|
# provides python3-foo for python3.XX-foo and vice versa for the default "main" version.
|
|
# E.g. in Fedora 33, python3.9-foo will provide python3-foo,
|
|
# python3-foo will provide python3.9-foo.
|
|
#
|
|
# There are two macros:
|
|
#
|
|
# This always contains the major.minor version (with dots), default for %%python3_version.
|
|
%__default_python3_version 3.12
|
|
#
|
|
# The pkgname version that determines the alternative provide name (e.g. python3.9-foo),
|
|
# set to the same as above, but historically hasn't included the dot.
|
|
# This is left intentionally a separate macro, in case the naming convention ever changes.
|
|
%__default_python3_pkgversion %__default_python3_version
|
|
|
|
# python3_pkgversion specifies the version of Python 3 in the distro.
|
|
# For Fedora, this is usually just "3".
|
|
# It can be a specific version distro-wide (e.g. "36" in EPEL7).
|
|
# Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks.
|
|
%python3_pkgversion 3
|
|
|
|
# Define the Python interpreter paths in the SRPM macros so that
|
|
# - they can be used in Build/Requires
|
|
# - they can be used in non-Python packages where requiring pythonX-devel would
|
|
# be an overkill
|
|
|
|
# use the underscored macros to redefine the behavior of %%python3_version etc.
|
|
%__python2 /usr/bin/python2
|
|
%__python3 /usr/bin/python%{python3_pkgversion}
|
|
|
|
# use the non-underscored macros to refer to Python in spec, etc.
|
|
%python2 %__python2
|
|
%python3 %__python3
|
|
|
|
# See https://fedoraproject.org/wiki/Changes/PythonMacroError
|
|
%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly}
|
|
|
|
# Users can use %%python only if they redefined %%__python (e.g. to %%__python3)
|
|
%python %__python
|
|
|
|
# Define where Python wheels will be stored and the prefix of -wheel packages
|
|
# - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that
|
|
# install packages into `/usr/share/python-wheels`. Both names are not
|
|
# versioned, because they're used by all Python 3 stacks.
|
|
# - In RHEL we want wheel packages named e.g. `python3-pip-wheel` and
|
|
# `python3.11-pip-wheel` that install packages into similarly versioned
|
|
# locations. We want each Python stack in RHEL to have their own wheels,
|
|
# because the main python3 wheels (which we can't upgrade) will likely be
|
|
# quite old by the time we're adding new alternate Python stacks.
|
|
# - In ELN we want to follow Fedora, because builds for ELN and Fedora rawhide
|
|
# need to be interoperable.
|
|
%python_wheel_pkg_prefix python%{?rhel:%{!?eln:%{python3_pkgversion}}}
|
|
%python_wheel_dir %{_datadir}/%{python_wheel_pkg_prefix}-wheels
|
|
|
|
|
|
### BRP scripts (and related macros)
|
|
|
|
## Automatically compile python files
|
|
%py_auto_byte_compile 1
|
|
## Should python bytecompilation errors terminate a build?
|
|
%_python_bytecompile_errors_terminate_build 1
|
|
## Should python bytecompilation compile outside python specific directories?
|
|
## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3
|
|
%_python_bytecompile_extra 0
|
|
## Helper macro to unset $SOURCE_DATE_EPOCH if %%clamp_mtime_to_source_date_epoch is not set
|
|
## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode
|
|
%__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"]
|
|
|
|
## Automatically apply __brp_fix_pyc_reproducibility
|
|
%py_fix_reproducibility 1
|
|
|
|
## The individual BRP scripts
|
|
%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}"
|
|
%__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility
|
|
%__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink
|
|
|
|
## This macro is included in redhat-rpm-config's %%__os_install_post
|
|
# Note that the order matters:
|
|
# 1. brp-python-bytecompile can create (or replace) pyc files
|
|
# 2. brp-fix-pyc-reproducibility can modify the pyc files from above
|
|
# 3. brp-python-hardlink de-duplicates identical pyc files
|
|
%__os_install_post_python \
|
|
%{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \
|
|
%{?py_fix_reproducibility:%{__brp_fix_pyc_reproducibility} -a "%{buildroot}%{_prefix}"} \
|
|
%{?py_reproducible_pyc_path:%{__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \
|
|
%{?__brp_python_hardlink} \
|
|
%{nil}
|
|
|
|
|
|
# === Macros for Build/Requires tags using Python dist tags ===
|
|
# - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages
|
|
# - These macros need to be in macros.python-srpm, because BuildRequires tags
|
|
# get rendered as runtime requires into the metadata of SRPMs.
|
|
|
|
# Converts Python dist name to a canonical format
|
|
%py_dist_name() %{lua:\
|
|
name = rpm.expand("%{?1:%{1}}");\
|
|
canonical = string.gsub(string.lower(name), "[^%w%[%]]+", "-");\
|
|
print(canonical);\
|
|
}
|
|
|
|
# Creates Python 2 dist tag(s) after converting names to canonical format
|
|
# Needs to first put all arguments into a list, because invoking a different
|
|
# macro (%%py_dist_name) overwrites them
|
|
%py2_dist() %{lua:\
|
|
args = {}\
|
|
arg = 1\
|
|
while (true) do\
|
|
name = rpm.expand("%{?" .. arg .. ":%{" .. arg .. "}}");\
|
|
if (name == nil or name == '') then\
|
|
break\
|
|
end\
|
|
args[arg] = name\
|
|
arg = arg + 1\
|
|
end\
|
|
for arg, name in ipairs(args) do\
|
|
canonical = rpm.expand("%py_dist_name " .. name);\
|
|
print("python2dist(" .. canonical .. ") ");\
|
|
end\
|
|
}
|
|
|
|
# Creates Python 3 dist tag(s) after converting names to canonical format
|
|
# Needs to first put all arguments into a list, because invoking a different
|
|
# macro (%%py_dist_name) overwrites them
|
|
%py3_dist() %{lua:\
|
|
python3_pkgversion = rpm.expand("%python3_pkgversion");\
|
|
args = {}\
|
|
arg = 1\
|
|
while (true) do\
|
|
name = rpm.expand("%{?" .. arg .. ":%{" .. arg .. "}}");\
|
|
if (name == nil or name == '') then\
|
|
break\
|
|
end\
|
|
args[arg] = name\
|
|
arg = arg + 1\
|
|
end\
|
|
for arg, name in ipairs(args) do\
|
|
canonical = rpm.expand("%py_dist_name " .. name);\
|
|
print("python" .. python3_pkgversion .. "dist(" .. canonical .. ") ");\
|
|
end\
|
|
}
|
|
|
|
# Macro to replace overly complicated references to PyPI source files.
|
|
# Expands to the pythonhosted URL for a package
|
|
# Accepts zero to three arguments:
|
|
# 1: The PyPI project name, defaulting to %%srcname if it is defined, then
|
|
# %%pypi_name if it is defined, then just %%name.
|
|
# 2: The PYPI version, defaulting to %%version with tildes stripped.
|
|
# 3: The file extension, defaulting to "tar.gz". (A period will be added
|
|
# automatically.)
|
|
# Requires %%__pypi_url and %%__pypi_default_extension to be defined.
|
|
%__pypi_url https://files.pythonhosted.org/packages/source/
|
|
%__pypi_default_extension tar.gz
|
|
|
|
%pypi_source() %{lua:
|
|
local src = rpm.expand('%1')
|
|
local ver = rpm.expand('%2')
|
|
local ext = rpm.expand('%3')
|
|
local url = rpm.expand('%__pypi_url')
|
|
\
|
|
-- If no first argument, try %srcname, then %pypi_name, then %name
|
|
-- Note that rpm leaves macros unchanged if they are not defined.
|
|
if src == '%1' then
|
|
src = rpm.expand('%srcname')
|
|
end
|
|
if src == '%srcname' then
|
|
src = rpm.expand('%pypi_name')
|
|
end
|
|
if src == '%pypi_name' then
|
|
src = rpm.expand('%name')
|
|
end
|
|
\
|
|
-- If no second argument, use %version
|
|
if ver == '%2' then
|
|
ver = rpm.expand('%version'):gsub('~', '')
|
|
end
|
|
\
|
|
-- If no third argument, use the preset default extension
|
|
if ext == '%3' then
|
|
ext = rpm.expand('%__pypi_default_extension')
|
|
end
|
|
\
|
|
local first = string.sub(src, 1, 1)
|
|
\
|
|
print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext)
|
|
}
|
|
|
|
%py_provides() %{lua:
|
|
local python = require 'fedora.srpm.python'
|
|
local rhel = rpm.expand('%{?rhel}')
|
|
local name = rpm.expand('%1')
|
|
if name == '%1' then
|
|
rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}')
|
|
end
|
|
local evr = rpm.expand('%2')
|
|
if evr == '%2' then
|
|
evr = rpm.expand('%{?epoch:%{epoch}:}%{version}-%{release}')
|
|
end
|
|
print('Provides: ' .. name .. ' = ' .. evr .. '\\n')
|
|
local provides = python.python_altprovides(name, evr)
|
|
for i, provide in ipairs(provides) do
|
|
print('Provides: ' .. provide .. '\\n')
|
|
end
|
|
-- We only generate these Obsoletes on CentOS/RHEL to provide clean upgrade
|
|
-- path, e.g. python3-foo obsoletes python3.9-foo from previous RHEL.
|
|
-- In Fedora this is not needed as we don't ship ecosystem packages
|
|
-- for alternative Python interpreters.
|
|
if rhel ~= '' then
|
|
-- Create Obsoletes only if the name does not end in a parenthesis,
|
|
-- as Obsoletes can't include parentheses.
|
|
-- This most commonly happens when the name contains an isa.
|
|
if (string.sub(name, "-1") ~= ")") then
|
|
local obsoletes = python.python_altobsoletes(name, evr)
|
|
for i, obsolete in ipairs(obsoletes) do
|
|
print('Obsoletes: ' .. obsolete .. '\\n')
|
|
end
|
|
end
|
|
end
|
|
}
|
|
|
|
%python_extras_subpkg(n:i:f:F) %{expand:%{lua:
|
|
local option_n = '-n (name of the base package)'
|
|
local option_i = '-i (buildroot path to metadata)'
|
|
local option_f = '-f (builddir path to a filelist)'
|
|
local option_F = '-F (skip %%files section)'
|
|
local value_n = rpm.expand('%{-n*}')
|
|
local value_i = rpm.expand('%{-i*}')
|
|
local value_f = rpm.expand('%{-f*}')
|
|
local value_F = rpm.expand('%{-F}')
|
|
local args = rpm.expand('%{*}')
|
|
if value_n == '' then
|
|
rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}')
|
|
end
|
|
if value_i == '' and value_f == '' and value_F == '' then
|
|
rpm.expand('%{error:%%%0: missing option ' .. option_i .. ' or ' .. option_f .. ' or ' .. option_F .. '}')
|
|
end
|
|
if value_i ~= '' and value_f ~= '' then
|
|
rpm.expand('%{error:%%%0: simultaneous ' .. option_i .. ' and ' .. option_f .. ' options are not possible}')
|
|
end
|
|
if value_i ~= '' and value_F ~= '' then
|
|
rpm.expand('%{error:%%%0: simultaneous ' .. option_i .. ' and ' .. option_F .. ' options are not possible}')
|
|
end
|
|
if value_f ~= '' and value_F ~= '' then
|
|
rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}')
|
|
end
|
|
if args == '' then
|
|
rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}')
|
|
end
|
|
local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}'
|
|
for extras in args:gmatch('[^%s,]+') do
|
|
local rpmname = value_n .. '+' .. extras
|
|
local pkgdef = '%package -n ' .. rpmname
|
|
local summary = 'Summary: Metapackage for ' .. value_n .. ': ' .. extras .. ' extras'
|
|
local description = '%description -n ' .. rpmname .. '\\\n'
|
|
local current_line = 'This is a metapackage bringing in'
|
|
for _, word in ipairs({extras, 'extras', 'requires', 'for', value_n .. '.'}) do
|
|
local line = current_line .. ' ' .. word
|
|
if line:len() > 79 then
|
|
description = description .. current_line .. '\\\n'
|
|
current_line = word
|
|
else
|
|
current_line = line
|
|
end
|
|
end
|
|
description = description .. current_line .. '\\\n' ..
|
|
'It makes sure the dependencies are installed.\\\n'
|
|
local files = ''
|
|
if value_i ~= '' then
|
|
files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i
|
|
elseif value_f ~= '' then
|
|
files = '%files -n ' .. rpmname .. ' -f ' .. value_f
|
|
end
|
|
for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do
|
|
print(line .. '\\\n')
|
|
end
|
|
end
|
|
}}
|