diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility new file mode 100644 index 0000000..05857b3 --- /dev/null +++ b/brp-fix-pyc-reproducibility @@ -0,0 +1,18 @@ +#!/bin/bash -eu + +# If using normal root, avoid changing anything. +if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then + exit 0 +fi + +# Defined as %py_reproducible_pyc_path macro and passed here as +# the first command-line argument +path_to_fix=${1:?} + +# First, check that the parser is available: +if [[ ! -x /usr/bin/marshalparser ]]; then + echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" + exit 1 +fi + +find "$path_to_fix" -type f -name '*.pyc' -exec /usr/bin/marshalparser --fix --overwrite '{}' '+' diff --git a/brp-python-bytecompile b/brp-python-bytecompile new file mode 100644 index 0000000..1911fa1 --- /dev/null +++ b/brp-python-bytecompile @@ -0,0 +1,150 @@ +#!/bin/bash +errors_terminate=$2 + +# Usage of %_python_bytecompile_extra is not allowed anymore +# See: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 +# Therefore $1 ($default_python) is not needed and is invoked with "" by default. +# $default_python stays in the arguments for backward compatibility and $extra for the following check: +extra=$3 +if [[ 0"$extra" -eq 1 ]]; then + echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr + exit 1 +fi + +compileall_flags="$4" + +# If using normal root, avoid changing anything. +if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then + exit 0 +fi + +# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +function python_clamp_source_mtime() +{ + local _=$1 + local python_binary=$2 + local _=$3 + local python_libdir="$4" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir" +} + +# This function now implements Python byte-compilation in three different ways: +# Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 +# In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. +# Python < 3.4 (inc. Python 2) uses compileall module from stdlib with some hacks +function python_bytecompile() +{ + local options=$1 + local python_binary=$2 + # local exclude=$3 # No longer used + local python_libdir="$4" + local compileall_flags="$5" + + python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") + + # + # Python 3.4 and higher + # + if [[ "$python_version" -ge 34 ]]; then + + # We compile all opt levels in one go: only when $options is empty. + if [[ -n "$options" ]]; then + return + fi + + if [[ "$python_version" -ge 39 ]]; then + # For Pyhon 3.9+, use the standard library + compileall_module=compileall + else + # For older Pythons, use compileall2 + compileall_module=compileall2 + fi + + if [[ "$python_version" -ge 37 ]]; then + # Force the TIMESTAMP invalidation mode + invalidation_option=--invalidation-mode=timestamp + else + # For older Pythons, the option does not exist + # as the invalidation is always based on size+mtime + invalidation_option= + fi + + # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib + # -o 0 -o 1 are the optimization levels + # -q disables verbose output + # -f forces the process to overwrite existing compiled files + # -e excludes symbolic links pointing outside the build root + # -s strips $RPM_BUILD_ROOT from the path + # -p prepends the leading slash to the path to make it absolute + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + + else +# +# Python 3.3 and lower (incl. Python 2) +# + +local real_libdir=${python_libdir/$RPM_BUILD_ROOT/} + +cat << EOF | $python_binary $options +import compileall, sys, os, re + +python_libdir = "$python_libdir" +depth = sys.getrecursionlimit() +real_libdir = "$real_libdir" +build_root = "$RPM_BUILD_ROOT" + +class Filter: + def search(self, path): + ret = not os.path.realpath(path).startswith(build_root) + return ret + +sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) +EOF + +fi +} + +# .pyc/.pyo files embed a "magic" value, identifying the ABI version of Python +# bytecode that they are for. +# +# The files below RPM_BUILD_ROOT could be targeting multiple versions of +# python (e.g. a single build that emits several subpackages e.g. a +# python26-foo subpackage, a python31-foo subpackage etc) +# +# Support this by assuming that below each /usr/lib/python$VERSION/, all +# .pyc/.pyo files are to be compiled for /usr/bin/python$VERSION. +# +# For example, below /usr/lib/python2.6/, we're targeting /usr/bin/python2.6 +# and below /usr/lib/python3.1/, we're targeting /usr/bin/python3.1 + +# Disable Python hash seed randomization +# This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +# Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) +export PYTHONHASHSEED=0 + +shopt -s nullglob +find "$RPM_BUILD_ROOT" -type d -print0|grep -z -E "/(usr|app)/lib(64)?/python[0-9]\.[0-9]+$" | while read -d "" python_libdir; +do + python_binary=$(basename "$python_libdir") + echo "Bytecompiling .py files below $python_libdir using $python_binary" + + # Generate normal (.pyc) byte-compiled files. + python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then + # One or more of the files had inaccessible mtime + exit 1 + fi + python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then + # One or more of the files had a syntax error + exit 1 + fi + + # Generate optimized (.pyo) byte-compiled files. + # N.B. For Python 3.4+, this call does nothing + python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" + if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then + # One or more of the files had a syntax error + exit 1 + fi +done diff --git a/brp-python-hardlink b/brp-python-hardlink new file mode 100644 index 0000000..5fd1b43 --- /dev/null +++ b/brp-python-hardlink @@ -0,0 +1,25 @@ +#!/bin/sh + +# If using normal root, avoid changing anything. +if [ -z "$RPM_BUILD_ROOT" ] || [ "$RPM_BUILD_ROOT" = "/" ]; then + exit 0 +fi + +hardlink_if_same() { + if cmp -s "$1" "$2" ; then + ln -f "$1" "$2" + return 0 + fi + return 1 +} + +# Hardlink identical *.pyc, *.pyo, and *.opt-[12].pyc. +# Originally from PLD's rpm-build-macros +find "$RPM_BUILD_ROOT" -type f -name "*.pyc" -not -name "*.opt-[12].pyc" | while read pyc ; do + hardlink_if_same "$pyc" "${pyc%c}o" + o1pyc="${pyc%pyc}opt-1.pyc" + hardlink_if_same "$pyc" "$o1pyc" + o2pyc="${pyc%pyc}opt-2.pyc" + hardlink_if_same "$pyc" "$o2pyc" || hardlink_if_same "$o1pyc" "$o2pyc" +done +exit 0 diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo new file mode 100755 index 0000000..b72d704 --- /dev/null +++ b/brp-python-rpm-in-distinfo @@ -0,0 +1,15 @@ +#!/usr/bin/bash + +set -eu +# If using normal root, avoid changing anything. +if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then + exit 0 +fi + +find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do + if cmp -s <(echo pip) "$installer" ; then + echo "rpm" > "$installer" + rm -f "$(dirname "$installer")/RECORD" + fi +done +exit 0 diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py new file mode 100644 index 0000000..1d03a6b --- /dev/null +++ b/clamp_source_mtime.py @@ -0,0 +1,163 @@ +"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH + +When called as a script with arguments, this compiles the directories +given as arguments recursively. + +If upstream is interested, this can be later integrated to the compileall module +as an additional option (e.g. --clamp-source-mtime). + +License: +This has been derived from the Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ +""" +from __future__ import print_function +import os +import sys + +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) + +__all__ = ["clamp_dir", "clamp_file"] + + +def _walk_dir(dir, maxlevels, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + for result in _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet): + yield result + + +def clamp_dir(dir, source_date_epoch, quiet=0): + """Clamp the mtime of all modules in the given directory tree. + + Arguments: + + dir: the directory to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + maxlevels = sys.getrecursionlimit() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + for file in files: + if not clamp_file(file, source_date_epoch, quiet=quiet): + success = False + return success + + +def clamp_file(fullname, source_date_epoch, quiet=0): + """Clamp the mtime of one file. + + Arguments: + + fullname: the file to byte-compile + source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH + quiet: full output with False or 0, errors only with 1, + no output with 2 + """ + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + if os.path.isfile(fullname) and not os.path.islink(fullname): + if name[-3:] == '.py': + try: + mtime = int(os.stat(fullname).st_mtime) + atime = int(os.stat(fullname).st_atime) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error checking mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + if mtime > source_date_epoch: + if not quiet: + print('Clamping mtime of {!r}'.format(fullname)) + try: + os.utime(fullname, (atime, source_date_epoch)) + except OSError as e: + if quiet >= 2: + return False + elif quiet: + print('*** Error clamping mtime of {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + return False + return True + + +def main(): + """Script main program.""" + import argparse + + source_date_epoch = os.getenv('SOURCE_DATE_EPOCH') + if not source_date_epoch: + print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set") + return True # This is a success, no action needed + try: + source_date_epoch = int(source_date_epoch) + except ValueError: + print("$SOURCE_DATE_EPOCH must be an integer") + return False + + parser = argparse.ArgumentParser( + description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+', + help=('zero or more file and directory paths ' + 'to clamp')) + + args = parser.parse_args() + clamp_dests = args.clamp_dest + + success = True + try: + for dest in clamp_dests: + if os.path.isfile(dest): + if not clamp_file(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + else: + if not clamp_dir(dest, quiet=args.quiet, + source_date_epoch=source_date_epoch): + success = False + return success + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/compileall2.py b/compileall2.py index c58e545..ea7e76f 100644 --- a/compileall2.py +++ b/compileall2.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, if compiles all modules on sys.path, without +Without arguments, it compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -36,11 +36,11 @@ PY35 = sys.version_info[0:2] >= (3, 5) # introduced in Python 3.7. These cases are covered by variables here or by PY37 # variable itself. if PY37: - pyc_struct_format = '<4sll' + pyc_struct_format = '<4sLL' pyc_header_lenght = 12 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) else: - pyc_struct_format = '<4sl' + pyc_struct_format = '<4sL' pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) @@ -106,7 +106,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path @@ -120,23 +120,34 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, stripdir = dir prependdir = ddir ddir = None - if workers is not None: - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - elif workers != 1: - try: - # Only import when needed, as low resource platforms may - # fail to import it - from concurrent.futures import ProcessPoolExecutor - except ImportError: - workers = 1 + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + if workers != 1: + # Check if this is a system where ProcessPoolExecutor can function. + from concurrent.futures.process import _check_system_limits + try: + _check_system_limits() + except NotImplementedError: + workers = 1 + else: + from concurrent.futures import ProcessPoolExecutor if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True - if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + if workers != 1 and ProcessPoolExecutor is not None: + mp_context_arg = {} + if PY37: + import multiprocessing + if multiprocessing.get_start_method() == 'fork': + mp_context = multiprocessing.get_context('forkserver') + else: + mp_context = None + mp_context_arg = {"mp_context": mp_context} + # If workers == 0, let ProcessPoolExecutor choose workers = workers or None - with ProcessPoolExecutor(max_workers=workers) as executor: + with ProcessPoolExecutor(max_workers=workers, + **mp_context_arg) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -178,7 +189,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beggining of original file path, applied + prependdir: path to prepend to beginning of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. @@ -190,10 +201,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) - else: - fullname = str(fullname) + fullname = os.fspath(fullname) + stripdir = os.fspath(stripdir) if stripdir is not None else None name = os.path.basename(fullname) dfile = None @@ -206,13 +215,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) - ddir_parts = list(fullname_parts) - for spart, opart in zip(stripdir_parts, fullname_parts): - if spart == opart: - ddir_parts.remove(spart) - - dfile = os.path.join(*ddir_parts) + if stripdir_parts != fullname_parts[:len(stripdir_parts)]: + if quiet < 2: + print("The stripdir path {!r} is not a valid prefix for " + "source path {!r}; ignoring".format(stripdir, fullname)) + else: + dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) if prependdir is not None: if dfile is None: @@ -258,7 +267,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack(*(pyc_header_format + (mtime,))) + expect = struct.pack(*(pyc_header_format + (mtime & 0xFFFF_FFFF,))) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(pyc_header_lenght) @@ -301,9 +310,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, - errors='backslashreplace') - msg = msg.decode(sys.stdout.encoding) + encoding = sys.stdout.encoding or sys.getdefaultencoding() + msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False @@ -408,8 +416,8 @@ def main(): type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses optimization level of ' - 'Python interpreter itself (specified by -O).')) + 'Default is -1 which uses the optimization level ' + 'of the Python interpreter itself (see -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', @@ -456,7 +464,8 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + with (sys.stdin if args.flist=='-' else + open(args.flist, encoding="utf-8")) as f: for line in f: compile_dests.append(line.strip()) except OSError: @@ -464,9 +473,6 @@ def main(): print("Error reading file list {}".format(args.flist)) return False - if args.workers is not None: - args.workers = args.workers or None - if PY37 and args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] diff --git a/import_all_modules.py b/import_all_modules.py index 3930236..97e924c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,6 +7,7 @@ import os import re import site import sys +import traceback from contextlib import contextmanager from pathlib import Path @@ -93,11 +94,24 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. + + Return a list of failed modules. ''' + failed_modules = [] + for module in modules: print('Check import:', module, file=sys.stderr) - importlib.import_module(module) + try: + importlib.import_module(module) + except Exception: + traceback.print_exc(file=sys.stderr) + failed_modules.append(module) + + if failed_modules: + print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) + + return failed_modules def argparser(): @@ -164,7 +178,10 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - import_modules(modules) + failed_modules = import_modules(modules) + + if failed_modules: + raise SystemExit(1) if __name__ == '__main__': diff --git a/macros.pybytecompile b/macros.pybytecompile index 7bd555b..e81ddae 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -4,20 +4,27 @@ # Which unfortunately makes the definition more complicated than it should be # Usage: -# %py_byte_compile +# %%py_byte_compile # Example: -# %py_byte_compile %{__python3} %{buildroot}%{_datadir}/spam/plugins/ +# %%py_byte_compile %%{__python3} %%{buildroot}%%{_datadir}/spam/plugins/ # This will terminate build on SyntaxErrors, if you want to avoid that, # use it in a subshell like this: -# (%{py_byte_compile }) || : +# (%%{py_byte_compile }) || : # Setting PYTHONHASHSEED=0 disables Python hash seed randomization # This should help with byte-compilation reproducibility: https://bugzilla.redhat.com/show_bug.cgi?id=1686078 +# Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ +clamp_source_mtime () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \ +}\ +\ py2_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ failure=0\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ @@ -25,28 +32,38 @@ py2_byte_compile () {\ test $failure -eq 0\ }\ \ -py3_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ +py34_byte_compile () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ +}\ +py37_byte_compile () {\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ + bytecode_compilation_path="%2"\ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ - python_binary="env PYTHONHASHSEED=0 %1"\ + python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / $bytecode_compilation_path \ + $python_binary -s -B -m compileall %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ +# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \ +clamp_source_mtime "%1" "%2"; \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ # and it was merged back to stdlib in Python >= 3.9 \ +# Only Python 3.7+ supports and needs the --invalidation-mode option \ if [ "$python_version" -ge 39 ]; then \ py39_byte_compile "%1" "%2"; \ +elif [ "$python_version" -ge 37 ]; then \ +py37_byte_compile "%1" "%2"; \ elif [ "$python_version" -ge 34 ]; then \ -py3_byte_compile "%1" "%2"; \ +py34_byte_compile "%1" "%2"; \ else \ py2_byte_compile "%1" "%2"; \ fi diff --git a/macros.python b/macros.python index a773e57..4bd0da6 100644 --- a/macros.python +++ b/macros.python @@ -1,62 +1,109 @@ +# Memoize a macro to avoid calling the same expensive code multiple times in +# the specfile. +# There is no error handling, +# memoizing an undefined macro (or using such a key) has undefined behavior. +# Options: +# -n - The name of the macro to wrap +# -k - The name of the macro to use as a cache key +%_python_memoize(n:k:) %{lua: +local name = opt.n +-- NB: We use rpm.expand() here instead of the macros table to make sure errors +-- are propogated properly. +local cache_key = rpm.expand("%{" .. opt.k .. "}") +if not _python_macro_cache then + -- This is intentionally a global lua table + _python_macro_cache = {} +end +if not _python_macro_cache[cache_key] then + _python_macro_cache[cache_key] = {} +end +if not _python_macro_cache[cache_key][name] then + _python_macro_cache[cache_key][name] = rpm.expand("%{" .. name .. "}") +end +print(_python_macro_cache[cache_key][name]) +} + +# Deprecation wrapper, warns only once per macro +# Options: +# -n - The name of the macro that is deprecated +%_python_deprecated(n:) %{lua: +if not _python_deprecated_warned then + -- This is intentionally a global lua table + _python_deprecated_warned = {} +end +if not _python_deprecated_warned[opt.n] then + _python_deprecated_warned[opt.n] = true + local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. + "See the current Python packaging guidelines: " .. + "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" + macros.warn({msg}) +end +} + # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros -%python_sitelib %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +# platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks +%__python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitelib %{_python_memoize -n __python_sitelib -k __python} + +%__python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitearch %{_python_memoize -n __python_sitearch -k __python} + +%__python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version %{_python_memoize -n __python_version -k __python} + +%__python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_version_nodots %{_python_memoize -n __python_version_nodots -k __python} + +%__python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform %{_python_memoize -n __python_platform -k __python} + +%__python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_platform_triplet %{_python_memoize -n __python_platform_triplet -k __python} + +%__python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_ext_suffix %{_python_memoize -n __python_ext_suffix -k __python} + +%__python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") +%python_cache_tag %{_python_memoize -n __python_cache_tag -k __python} %py_setup setup.py -%py_shbang_opts -s +%_py_shebang_s s +%__py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py_shebang_P %{_python_memoize -n __py_shebang_P -k __python} +%py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python_version}.py ]; then - pathfix=/usr/bin/pathfix%{python_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py_shebang_flags}" fi - $pathfix -pni %{__python} $shebang_flags} + %{__python} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{expand:\\\ +%py_build() %{_python_deprecated -n py_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - -%py_build_wheel() %{expand:\\\ +%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{expand:\\\ +%py_install() %{_python_deprecated -n py_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python_sitelib} - %{__python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} + %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py_install_wheel() %{expand:\\\ - %{__python} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location + %{__python} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python_sitelib}/*.dist-info %{buildroot}%{python_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then @@ -99,6 +146,7 @@ local package = rpm.expand("%{?1}") local vr = rpm.expand("%{?epoch:%{epoch}:}%{version}-%{release}") local provides = python.python_altprovides(package, vr) + local default_python3_pkgversion = rpm.expand("%{__default_python3_pkgversion}") if (string.starts(package, "python3-")) then for i, provide in ipairs(provides) do print("\\nProvides: " .. provide) @@ -109,14 +157,14 @@ print(string.sub(package,9,string.len(package))) print(" < " .. vr) end - elseif (string.starts(package, "python" .. rpm.expand("%{__default_python3_pkgversion}") .. "-")) then + elseif (string.starts(package, "python" .. default_python3_pkgversion .. "-")) then for i, provide in ipairs(provides) do print("\\nProvides: " .. provide) end --Obsoleting the previous default python package (if it doesn't have isa) if (string.sub(package, "-1") ~= ")") then print("\\nObsoletes: python-") - print(string.sub(package,11,string.len(package))) + print(string.sub(package,8+string.len(default_python3_pkgversion),string.len(package))) print(" < " .. vr) end elseif (string.starts(package, "python")) then @@ -130,6 +178,16 @@ end } +# Environment variables for testing used standalone, e.g.: +# %%{py_test_envvars} %%{python} -m unittest +%py_test_envvars %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} + %python_disable_dependency_generator() \ %undefine __pythondist_requires \ %{nil} diff --git a/macros.python-srpm b/macros.python-srpm index ecd08e7..d5cd4e5 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -1,22 +1,3 @@ -# Define the Python interpreter paths in the SRPM macros so that -# - they can be used in Build/Requires -# - they can be used in non-Python packages where requiring pythonX-devel would -# be an overkill - -# use the underscored macros to redefine the behavior of %%python3_version etc. -%__python2 /usr/bin/python2 -%__python3 /usr/bin/python3 - -# use the non-underscored macros to refer to Python in spec, etc. -%python2 %__python2 -%python3 %__python3 - -# See https://fedoraproject.org/wiki/Changes/PythonMacroError -%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} - -# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) -%python %__python - # There are multiple Python 3 versions packaged, but only one can be the "main" version # That means that it owns the "python3" namespace: # - python3 package name @@ -36,21 +17,37 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.9 +%__default_python3_version 3.14 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. # This is left intentionally a separate macro, in case the naming convention ever changes. %__default_python3_pkgversion %__default_python3_version -# python3_pkgversion specifies the version of Python 3 in the distro. It can be -# a specific version (e.g. 34 in Fedora EPEL7) +# python3_pkgversion specifies the version of Python 3 in the distro. +# For Fedora, this is usually just "3". +# It can be a specific version distro-wide (e.g. "36" in EPEL7). +# Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 -# Set to /bin/true to avoid %ifdefs and %{? in specfiles -%__python3_other /bin/true -%py3_other_build /bin/true -%py3_other_install /bin/true +# Define the Python interpreter paths in the SRPM macros so that +# - they can be used in Build/Requires +# - they can be used in non-Python packages where requiring pythonX-devel would +# be an overkill + +# use the underscored macros to redefine the behavior of %%python3_version etc. +%__python2 /usr/bin/python2 +%__python3 /usr/bin/python%{python3_pkgversion} + +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 +%python3 %__python3 + +# See https://fedoraproject.org/wiki/Changes/PythonMacroError +%__python %{error:attempt to use unversioned python, define %%__python to %{__python2} or %{__python3} explicitly} + +# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) +%python %__python # Define where Python wheels will be stored and the prefix of -wheel packages # - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that @@ -67,6 +64,40 @@ %python_wheel_dir %{_datadir}/%{python_wheel_pkg_prefix}-wheels +### BRP scripts (and related macros) + +## Modifies installation method in .dist-info/INSTALLER file to rpm +%python_rpm_in_distinfo 1 +## Automatically compile python files +%py_auto_byte_compile 1 +## Should python bytecompilation errors terminate a build? +%_python_bytecompile_errors_terminate_build 1 +## Should python bytecompilation compile outside python specific directories? +## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 +%_python_bytecompile_extra 0 +## Helper macro to unset $SOURCE_DATE_EPOCH if %%clamp_mtime_to_source_date_epoch is not set +## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode +%__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] + +## The individual BRP scripts +%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo +%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" +%__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink + +## This macro is included in redhat-rpm-config's %%__os_install_post +# Note that the order matters: +# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file +# 2. brp-python-bytecompile can create (or replace) pyc files +# 3. brp-fix-pyc-reproducibility can modify the pyc files from above +# 4. brp-python-hardlink de-duplicates identical pyc files +%__os_install_post_python \ + %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ + %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ + %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ + %{?__brp_python_hardlink} \ +%{nil} + # === Macros for Build/Requires tags using Python dist tags === # - https://fedoraproject.org/wiki/Changes/Automatic_Provides_for_Python_RPM_Packages @@ -82,7 +113,7 @@ # Creates Python 2 dist tag(s) after converting names to canonical format # Needs to first put all arguments into a list, because invoking a different -# macro (%py_dist_name) overwrites them +# macro (%%py_dist_name) overwrites them %py2_dist() %{lua:\ args = {}\ arg = 1\ @@ -102,7 +133,7 @@ # Creates Python 3 dist tag(s) after converting names to canonical format # Needs to first put all arguments into a list, because invoking a different -# macro (%py_dist_name) overwrites them +# macro (%%py_dist_name) overwrites them %py3_dist() %{lua:\ python3_pkgversion = rpm.expand("%python3_pkgversion");\ args = {}\ @@ -124,12 +155,12 @@ # Macro to replace overly complicated references to PyPI source files. # Expands to the pythonhosted URL for a package # Accepts zero to three arguments: -# 1: The PyPI project name, defaulting to %srcname if it is defined, then -# %pypi_name if it is defined, then just %name. -# 2: The PYPI version, defaulting to %version with tildes stripped. +# 1: The PyPI project name, defaulting to %%srcname if it is defined, then +# %%pypi_name if it is defined, then just %%name. +# 2: The PYPI version, defaulting to %%version with tildes stripped. # 3: The file extension, defaulting to "tar.gz". (A period will be added # automatically.) -# Requires %__pypi_url and %__pypi_default_extension to be defined. +# Requires %%__pypi_url and %%__pypi_default_extension to be defined. %__pypi_url https://files.pythonhosted.org/packages/source/ %__pypi_default_extension tar.gz @@ -168,6 +199,7 @@ %py_provides() %{lua: local python = require 'fedora.srpm.python' + local rhel = rpm.expand('%{?rhel}') local name = rpm.expand('%1') if name == '%1' then rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}') @@ -181,17 +213,37 @@ for i, provide in ipairs(provides) do print('Provides: ' .. provide .. '\\n') end + -- We only generate these Obsoletes on CentOS/RHEL to provide clean upgrade + -- path, e.g. python3-foo obsoletes python3.9-foo from previous RHEL. + -- In Fedora this is not needed as we don't ship ecosystem packages + -- for alternative Python interpreters. + if rhel ~= '' then + -- Create Obsoletes only if the name does not end in a parenthesis, + -- as Obsoletes can't include parentheses. + -- This most commonly happens when the name contains an isa. + if (string.sub(name, "-1") ~= ")") then + local obsoletes = python.python_altobsoletes(name, evr) + for i, obsolete in ipairs(obsoletes) do + print('Obsoletes: ' .. obsolete .. '\\n') + end + end + end } -%python_extras_subpkg(n:i:f:F) %{expand:%{lua: +%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' local option_F = '-F (skip %%files section)' + local option_a = '-a (insert BuildArch: noarch)' + local option_A = '-A (do not insert BuildArch: noarch (default))' local value_n = rpm.expand('%{-n*}') local value_i = rpm.expand('%{-i*}') local value_f = rpm.expand('%{-f*}') local value_F = rpm.expand('%{-F}') + local value_a = rpm.expand('%{-a}') + local value_A = rpm.expand('%{-A}') + local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -208,10 +260,14 @@ if value_f ~= '' and value_F ~= '' then rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') end + if value_a ~= '' and value_A ~= '' then + rpm.expand('%{error:%%%0: simultaneous ' .. option_a .. ' and ' .. option_A .. ' options are not possible}') + end if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' + local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname @@ -231,11 +287,15 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end - for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do + local tags = summary .. '\\\n' .. requires + if value_a ~= '' then + tags = tags .. '\\\nBuildArch: noarch' + end + for i, line in ipairs({pkgdef, tags, description, files, ''}) do print(line .. '\\\n') end end diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom new file mode 100644 index 0000000..41389f9 --- /dev/null +++ b/macros.python-wheel-sbom @@ -0,0 +1,124 @@ +# The macros in this file are used to add SBOM to wheel files that we ship. +# Majority of Python packages will not need to do that, +# as they only use wheels as an intermediate artifact. +# The macros will be used by packages installing wheel to %%python_wheel_dir +# or by Python interpreters bundling their own (patched) wheels. +# +# The runtime dependencies are not Required by the python-rpm-macros package, +# users of this macro need to specify them on their own or rely on the fact that +# they are all available in the default buildroot. +# +# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS +# +# The wheels are modified in-place. + + +# Path of the SBOM file in the PEP 770 .dist-info/sboms directory +# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ +# section Recognized file patterns +%__python_wheel_sbom_filename bom.json + + +# The SBOM content to put to the file +# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 +%__python_wheel_sbom_content %{expand:{ + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "components": [ + { + "type": "library", + "name": "%{name}", + "version": "%{version}-%{release}", + "purl": "%{__python_wheel_purl}" + } + ] +}} + + +# The purl used above +# We use the src package name (which is easier to get and more useful to consumers). +# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 +# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ +%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src + + +# The purl namespace used above +# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ +# Distributors, define %%dist_purl_namespace to set this. +# The rest of the code is fallback for distributions without it (relying on %%dist_name). +%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: + if macros.epel then + -- being epel beats the %%dist_name value + -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 + print("epel") + else + local dist_map = { + -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm + -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 + ["Fedora Linux"] = "fedora", + -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 + ["CentOS Stream"] = "centos", + -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ + ["Red Hat Enterprise Linux"] = "redhat", + -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html + ["AlmaLinux"] = "almalinux", + -- from https://github.com/google/osv.dev/pull/2939 + ["Rocky Linux"] = "rocky-linux", + } + print(dist_map[macros.dist_name] or "unknown") + end +}} + + +# A Bash scriptlet to inject the SBOM file into the wheel(s) +# The macro takes positional nargs+ with wheel paths +# For each wheel, it +# 1. aborts if the SBOM file is already there (it won't override) +# 2. inserts the SBOM file to .dist-info/sboms +# 3. amends .dist-info/RECORD with the added SBOM file +%python_wheel_inject_sbom() %{expand:( + %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] + + set -eu -o pipefail + export LANG=C.utf-8 + + tmpdir=$(mktemp -d) + trap 'rm -rf "$tmpdir"' EXIT + pwd0=$(pwd) + ret=0 + + for whl in %{*}; do + cd "$tmpdir" + if [[ "$whl" != /* ]]; then + whl="$pwd0/$whl" + fi + + record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') + distinfo="${record%%/RECORD}" + bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" + + if zipinfo -1 "$whl" | grep -qFx "$bom"; then + echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 + ret=1 + continue + fi + + unzip "$whl" "$record" + mkdir "$distinfo/sboms" + echo '%{__python_wheel_sbom_content}' > "$bom" + checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" + size="$(wc --bytes "$bom" | cut -f1 -d' ')" + echo "$bom,$checksum,$size" >> "$record" + + if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then + touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" + fi + + zip -r "$whl" "$record" "$bom" + rm -rf "$distinfo" + cd "$pwd0" + done + + exit $ret +)} + diff --git a/macros.python2 b/macros.python2 deleted file mode 100644 index d4d4eda..0000000 --- a/macros.python2 +++ /dev/null @@ -1,64 +0,0 @@ -%python2_sitelib %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python2_sitearch %(%{__python2} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python2_version %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python2_platform %(%{__python2} -Esc "import sysconfig; print(sysconfig.get_platform())") - -%py2_shbang_opts -s -%py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) -%py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-}) -%py2_shebang_fix %{expand:\\\ - if [ -z "%{?py_shebang_flags}" ]; then - shebang_flags="-k" - else - shebang_flags="-ka%{py2_shebang_flags}" - fi - /usr/bin/pathfix.py -pni %{__python2} $shebang_flags} - -# Use the slashes after expand so that the command starts on the same line as -# the macro -# The `sleep 1` commands work around a race in install; see: -# https://bugzilla.redhat.com/show_bug.cgi?id=1644923 -%py2_build() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} build --executable="%{__python2} %{py2_shbang_opts}" %{?*} - sleep 1 -} - -%py2_build_egg() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} bdist_egg %{?*} - sleep 1 -} - -%py2_build_wheel() %{expand:\\\ - sleep 1 - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} - sleep 1 -} - -%py2_install() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python2} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py2_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python2_sitelib} - %{__python2} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python2_version}.egg %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py2_install_wheel() %{expand:\\\ - %{__python2} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location - rm -rfv %{buildroot}%{_bindir}/__pycache__ - for distinfo in %{buildroot}%{python2_sitelib}/*.dist-info %{buildroot}%{python2_sitearch}/*.dist-info; do - if [ -f ${distinfo}/direct_url.json ]; then - rm -fv ${distinfo}/direct_url.json - sed -i '/direct_url.json/d' ${distinfo}/RECORD - fi - done -} diff --git a/macros.python3 b/macros.python3 index d545d8f..38cdcd4 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,60 +1,66 @@ -%python3_sitelib %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib())") -%python3_sitearch %(%{__python3} -Ic "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") -%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +# platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks +%__python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitelib %{_python_memoize -n __python3_sitelib -k __python3} + +%__python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitearch %{_python_memoize -n __python3_sitearch -k __python3} + +%__python3_version %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version %{_python_memoize -n __python3_version -k __python3} + +%__python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_version_nodots %{_python_memoize -n __python3_version_nodots -k __python3} + +%__python3_platform %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python3_platform %{_python_memoize -n __python3_platform -k __python3} + +%__python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_platform_triplet %{_python_memoize -n __python3_platform_triplet -k __python3} + +%__python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_ext_suffix %{_python_memoize -n __python3_ext_suffix -k __python3} + +%__python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; print(sys.implementation.cache_tag)") +%python3_cache_tag %{_python_memoize -n __python3_cache_tag -k __python3} + %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} -%py3_shbang_opts -s +%_py3_shebang_s s +%__py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") +%_py3_shebang_P %{_python_memoize -n __py3_shebang_P -k __python3} +%py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python3_version}.py ]; then - pathfix=/usr/bin/pathfix%{python3_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py3_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py3_shebang_flags}" fi - $pathfix -pni %{__python3} $shebang_flags} + %{__python3} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python3} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{expand:\\\ +%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_egg() %{expand:\\\ - CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} -} - -%py3_build_wheel() %{expand:\\\ +%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{expand:\\\ +%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ - %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} %{?*} - rm -rfv %{buildroot}%{_bindir}/__pycache__ -} - -%py3_install_egg() %{expand:\\\ - mkdir -p %{buildroot}%{python3_sitelib} - %{__python3} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} + %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } %py3_install_wheel() %{expand:\\\ - %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --no-deps --no-index --no-warn-script-location + %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ for distinfo in %{buildroot}%{python3_sitelib}/*.dist-info %{buildroot}%{python3_sitearch}/*.dist-info; do if [ -f ${distinfo}/direct_url.json ]; then @@ -100,16 +106,21 @@ pyminor = path:match("/python3.(%d+)/") or "*" dirname = path:match("(.*/)") modulename = path:match(".*/([^/]+).py") + -- %%python3_cache_tag is not used here because this macro supports not-installed CPythons print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") end } -# This is intended for Python 3 only, hence also no Python version in the name. -%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) -%pytest %{expand:\\\ +# Environment variables used by %%pytest, %%tox or standalone, e.g.: +# %%{py3_test_envvars} %%{python3} -m unittest +%py3_test_envvars %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - %__pytest} + PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) +%pytest %py3_test_envvars %__pytest diff --git a/pathfix.py b/pathfix.py new file mode 100644 index 0000000..6808382 --- /dev/null +++ b/pathfix.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 + +import sys +import os +from stat import * +import getopt + +err = sys.stderr.write +dbg = err +rep = sys.stdout.write + +new_interpreter = None +preserve_timestamps = False +create_backup = True +keep_flags = False +add_flags = b'' + + +def main(): + global new_interpreter + global preserve_timestamps + global create_backup + global keep_flags + global add_flags + + usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' % + sys.argv[0]) + try: + opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn') + except getopt.error as msg: + err(str(msg) + '\n') + err(usage) + sys.exit(2) + for o, a in opts: + if o == '-i': + new_interpreter = a.encode() + if o == '-p': + preserve_timestamps = True + if o == '-n': + create_backup = False + if o == '-k': + keep_flags = True + if o == '-a': + add_flags = a.encode() + if b' ' in add_flags: + err("-a option doesn't support whitespaces") + sys.exit(2) + if not new_interpreter or not new_interpreter.startswith(b'/') or \ + not args: + err('-i option or file-or-directory missing\n') + err(usage) + sys.exit(2) + bad = 0 + for arg in args: + if os.path.isdir(arg): + if recursedown(arg): bad = 1 + elif os.path.islink(arg): + err(arg + ': will not process symbolic links\n') + else: + if fix(arg): bad = 1 + sys.exit(bad) + + +def ispython(name): + return name.endswith('.py') + + +def recursedown(dirname): + dbg('recursedown(%r)\n' % (dirname,)) + bad = 0 + try: + names = os.listdir(dirname) + except OSError as msg: + err('%s: cannot list directory: %r\n' % (dirname, msg)) + return 1 + names.sort() + subdirs = [] + for name in names: + if name in (os.curdir, os.pardir): continue + fullname = os.path.join(dirname, name) + if os.path.islink(fullname): pass + elif os.path.isdir(fullname): + subdirs.append(fullname) + elif ispython(name): + if fix(fullname): bad = 1 + for fullname in subdirs: + if recursedown(fullname): bad = 1 + return bad + + +def fix(filename): +## dbg('fix(%r)\n' % (filename,)) + try: + f = open(filename, 'rb') + except IOError as msg: + err('%s: cannot open: %r\n' % (filename, msg)) + return 1 + with f: + line = f.readline() + fixed = fixline(line) + if line == fixed: + rep(filename+': no change\n') + return + head, tail = os.path.split(filename) + tempname = os.path.join(head, '@' + tail) + try: + g = open(tempname, 'wb') + except IOError as msg: + err('%s: cannot create: %r\n' % (tempname, msg)) + return 1 + with g: + rep(filename + ': updating\n') + g.write(fixed) + BUFSIZE = 8*1024 + while 1: + buf = f.read(BUFSIZE) + if not buf: break + g.write(buf) + + # Finishing touch -- move files + + mtime = None + atime = None + # First copy the file's mode to the temp file + try: + statbuf = os.stat(filename) + mtime = statbuf.st_mtime + atime = statbuf.st_atime + os.chmod(tempname, statbuf[ST_MODE] & 0o7777) + except OSError as msg: + err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) + # Then make a backup of the original file as filename~ + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) + # Now move the temp file to the original file + try: + os.rename(tempname, filename) + except OSError as msg: + err('%s: rename failed (%r)\n' % (filename, msg)) + return 1 + if preserve_timestamps: + if atime and mtime: + try: + os.utime(filename, (atime, mtime)) + except OSError as msg: + err('%s: reset of timestamp failed (%r)\n' % (filename, msg)) + return 1 + # Return success + return 0 + + +def parse_shebang(shebangline): + shebangline = shebangline.rstrip(b'\n') + start = shebangline.find(b' -') + if start == -1: + return b'' + return shebangline[start:] + + +def populate_flags(shebangline): + old_flags = b'' + if keep_flags: + old_flags = parse_shebang(shebangline) + if old_flags: + old_flags = old_flags[2:] + if not (old_flags or add_flags): + return b'' + # On Linux, the entire string following the interpreter name + # is passed as a single argument to the interpreter. + # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s" + # so shebang should have single '-' where flags are given and + # flag might need argument for that reasons adding new flags is + # between '-' and original flags + # e.g. #! /usr/bin/python3 -sW Error + return b' -' + add_flags + old_flags + + +def fixline(line): + if not line.startswith(b'#!'): + return line + + if b"python" not in line: + return line + + flags = populate_flags(line) + return b'#! ' + new_interpreter + flags + b'\n' + + +if __name__ == '__main__': + main() diff --git a/plan.fmf b/plan.fmf new file mode 100644 index 0000000..88850dd --- /dev/null +++ b/plan.fmf @@ -0,0 +1,40 @@ +execute: + how: tmt + +discover: + - name: same_repo + how: shell + tests: + - name: pytest + test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - name: manual_byte_compilation_clamp_mtime_off + path: /tests + test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec + - name: manual_byte_compilation_clamp_mtime_on + path: /tests + test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec + - name: rpmlint_clamp_mtime_off + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: rpmlint_clamp_mtime_on + test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 + - name: python_wheel_inject_sbom + path: /tests + test: rpmbuild -ba testwheel.spec + +prepare: + - name: Install dependencies + how: install + package: + - rpm-build + - rpmlint + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-setuptools + - python3-pip + - python3-pytest + - python3.6 + - dnf + - name: Update packages + how: shell + script: dnf upgrade -y diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c0eda7f..263853a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,25 +1,62 @@ Name: python-rpm-macros -Version: 3.9 -Release: 20%{?dist} Summary: The common Python RPM macros -# macros and lua: MIT, compileall2.py: PSFv2, import_all_modules.py: MIT -License: MIT and Python +URL: https://src.fedoraproject.org/rpms/python-rpm-macros/ # Macros: Source101: macros.python Source102: macros.python-srpm -Source103: macros.python2 Source104: macros.python3 Source105: macros.pybytecompile +Source106: macros.python-wheel-sbom # Lua files Source201: python.lua # Python code -%global compileall2_version 0.7.1 +%global compileall2_version 0.8.0 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py +%global pathfix_version 1.0.0 +Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py +Source304: clamp_source_mtime.py + +# BRP scripts +# This one is from redhat-rpm-config < 190 +# A new upstream is forming in https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-bytecompile +# But our version is riddled with Fedora-isms +# We might eventually move to upstream source + Fedora patches, but we are not there yet +Source401: brp-python-bytecompile +# This one is from https://github.com/rpm-software-management/python-rpm-packaging/blob/main/scripts/brp-python-hardlink +# But we don't use a link in case it changes in upstream, there are no "versions" there yet +# This was removed from RPM 4.17+ so we maintain it here instead +Source402: brp-python-hardlink +# This one is from redhat-rpm-config < 190 +# It has no upstream yet +Source403: brp-fix-pyc-reproducibility +# brp script to write "rpm" string into the .dist-info/INSTALLER file +Source404: brp-python-rpm-in-distinfo + +# macros and lua: MIT +# import_all_modules.py: MIT +# compileall2.py, clamp_source_mtime.py: PSF-2.0 +# pathfix.py: PSF-2.0 +# brp scripts: GPL-2.0-or-later +License: MIT AND PSF-2.0 AND GPL-2.0-or-later + +# The package version MUST be always the same as %%{__default_python3_version}. +# To have only one source of truth, we load the macro and use it. +# The macro is defined in python-srpm-macros. +%{lua: +if posix.stat(rpm.expand('%{SOURCE102}')) then + rpm.load(rpm.expand('%{SOURCE102}')) +elseif posix.stat('macros.python-srpm') then + -- something is parsing the spec without _sourcedir macro properly set + rpm.load('macros.python-srpm') +end +} +Version: %{__default_python3_version} +Release: 9%{?dist} BuildArch: noarch @@ -28,6 +65,12 @@ BuildArch: noarch # For compileall2.py Requires: python-srpm-macros = %{version}-%{release} +# The packages are called python(3)-(s)rpm-macros +# We never want python3-rpm-macros to provide python-rpm-macros +# We opt out from all Python name-based automatic provides and obsoletes +%undefine __pythonname_provides +%undefine __pythonname_obsoletes + %description This package contains the unversioned Python RPM macros, that most implementations should rely on. @@ -40,7 +83,8 @@ python?-devel packages require it. So install a python-devel package instead. Summary: RPM macros for building Python source packages # For directory structure and flags macros -Requires: redhat-rpm-config +# Versions before 190 contained some brp scripts moved into python-srpm-macros +Requires: redhat-rpm-config >= 190 # We bundle our own software here :/ Provides: bundled(python3dist(compileall2)) = %{compileall2_version} @@ -49,19 +93,6 @@ Provides: bundled(python3dist(compileall2)) = %{compileall2_version} RPM macros for building Python source packages. -%package -n python2-rpm-macros -Summary: RPM macros for building Python 2 packages - -# For %%__python2 and %%python2 -Requires: python-srpm-macros = %{version}-%{release} - -# For %%py_setup -Requires: python-rpm-macros = %{version}-%{release} - -%description -n python2-rpm-macros -RPM macros for building Python 2 packages. - - %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages @@ -79,6 +110,10 @@ RPM macros for building Python 3 packages. %autosetup -c -T cp -a %{sources} . +# We want to have shebang in the script upstream but not here so +# the package with macros does not depend on Python. +sed -i '1s=^#!/usr/bin/env python3==' pathfix.py + %install mkdir -p %{buildroot}%{rpmmacrodir} @@ -89,28 +124,184 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ + + +# We define our own BRPs here to use the ones from the %%{buildroot}, +# that way, this package can be built when it includes them for the first time. +# It also ensures that: +# - our BRPs can execute +# - if our BRPs affect this package, we don't need to build it twice +%define add_buildroot() %{lua:print((macros[macros[1]]:gsub(macros._rpmconfigdir, macros.buildroot .. macros._rpmconfigdir)))} +%global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} +%global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} +%global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} +%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} + + +%check +# no macros in comments +grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py +%{_rpmconfigdir}/redhat/pathfix.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/clamp_source_mtime.py +%{_rpmconfigdir}/redhat/brp-python-bytecompile +%{_rpmconfigdir}/redhat/brp-python-hardlink +%{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility +%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua -%files -n python2-rpm-macros -%{rpmmacrodir}/macros.python2 - %files -n python3-rpm-macros %{rpmmacrodir}/macros.python3 %changelog -* Mon Nov 01 2021 Karolina Surma - 3.9-20 +* Thu Oct 16 2025 Miro Hrončok - 3.14-9 +- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content +- That way, accidentally unpackaged files within are reported as errors + +* Tue Sep 09 2025 Miro Hrončok - 3.14-8 +- %%python_extras_subpkg: Add -v option to specify the required version(-release) +- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) + +* Fri Aug 29 2025 Miro Hrončok - 3.14-7 +- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos + +* Wed Aug 13 2025 Miro Hrončok - 3.14-6 +- Introduce %%python_wheel_inject_sbom + +* Mon Aug 11 2025 Lumír Balhar - 3.14-5 +- import_all_modules: Add error handling for import failures + +* Fri Jul 25 2025 Fedora Release Engineering +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + +* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 +- pathfix.py: Don't fail on symbolic links + +* Sun Jun 29 2025 Miro Hrončok - 3.14-2 +- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install +- Deprecate %%py_build, %%py_build_wheel, and %%py_install +- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros + +* Wed May 28 2025 Karolina Surma - 3.14-1 +- Update main Python to 3.14 + +* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 +- Add brp script to modify .dist-info/INSTALLER file + +* Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild + +* Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + +* Tue Jun 25 2024 Cristian Le - 3.13-2 +- %%python_extras_subpkg: Add option -a to include BuildArch: noarch + +* Thu Jun 06 2024 Karolina Surma - 3.13-1 +- Update main Python to 3.13 + +* Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 +- Minor improvements to brp-fix-pyc-reproducibility + +* Fri Mar 22 2024 Lumír Balhar - 3.12-8 +- Update bundled compileall2 to version 0.8.0 + +* Thu Jan 25 2024 Miro Hrončok - 3.12-7 +- %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set + +* Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Mon Oct 09 2023 Maxwell G - 3.12-5 +- Fix python macro memoizing to account for changing %%__python3 + +* Tue Sep 05 2023 Maxwell G - 3.12-4 +- Remove %%py3_build_egg and %%py3_install_egg macros. + +* Wed Aug 09 2023 Karolina Surma - 3.12-3 +- Declare the license as an SPDX expression + +* Fri Jul 21 2023 Fedora Release Engineering - 3.12-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + +* Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 +- Update main Python to Python 3.12 +- https://fedoraproject.org/wiki/Changes/Python3.12 + +* Thu Mar 16 2023 Miro Hrončok - 3.11-10 +- Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly +- Fixes: rhbz#2179149 + +* Fri Jan 20 2023 Miro Hrončok - 3.11-9 +- Memoize values of macros that execute python to get their value +- Fixes: rhbz#2155505 + +* Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + +* Mon Dec 19 2022 Miro Hrončok - 3.11-7 +- Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set +- Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) +- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes +- Bytecompilation: Compile Python files in parallel, according to %%_smp_mflags + +* Sun Nov 13 2022 Miro Hrončok - 3.11-6 +- Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest +- pytest-xdist 3+ respects this value when -n auto is used +- Expose the environment variables used by %%pytest via %%{py3_test_envvars} + +* Tue Oct 25 2022 Lumír Balhar - 3.11-5 +- Include pathfix.py in this package + +* Fri Jul 22 2022 Fedora Release Engineering - 3.10-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild + +* Tue Jul 19 2022 Miro Hrončok - 3.11-3 +- Add "P" to %%py3_shbang_opts, %%py3_shbang_opts_nodash, %%py3_shebang_flags + and to %%py_shbang_opts, %%py_shbang_opts_nodash, %%py_shebang_flags +- https://fedoraproject.org/wiki/Changes/PythonSafePath + +* Mon Jun 20 2022 Miro Hrončok - 3.11-2 +- Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 + +* Mon Jun 13 2022 Tomáš Hrnčiar - 3.11-1 +- Update main Python to Python 3.11 +- https://fedoraproject.org/wiki/Changes/Python3.11 + +* Thu May 26 2022 Owen Taylor - 3.10-18 +- Support installing to %%{_prefix} other than /usr + +* Tue Feb 08 2022 Tomas Orsava - 3.10-17 +- %%py_provides: Do not generate Obsoletes for names containing parentheses + +* Mon Jan 31 2022 Miro Hrončok - 3.10-16 +- Explicitly opt-out from Python name-based provides and obsoletes generators + +* Tue Dec 21 2021 Tomas Orsava - 3.10-15 +- Add lua helper functions to make it possible to automatically generate + Obsoletes tags +- Modify the %%py_provides macro to also generate Obsoletes tags on CentOS/RHEL + +* Wed Dec 08 2021 Miro Hrončok - 3.10-14 +- Set %%__python3 value according to %%python3_pkgversion + I.e. when %%python3_pkgversion is 3.12, %%__python3 is /usr/bin/python3.12 + +* Mon Nov 01 2021 Karolina Surma - 3.10-13 - Fix multiline arguments processing for %%py_check_import Resolves: rhbz#2018809 - Fix %%py_shebang_flags handling within %%py_check_import @@ -119,28 +310,69 @@ Resolves: rhbz#2018615 Resolves: rhbz#2018551 - Move import_all_modules.py from python-srpm-macros to python-rpm-macros -* Wed Oct 27 2021 Karolina Surma - 3.9-19 +* Mon Oct 25 2021 Karolina Surma - 3.10-12 - Introduce -f (read from file) option to %%py{3}_check_import - Introduce -t (filter top-level modules) option to %%py{3}_check_import - Introduce -e (exclude module globs) option to %%py{3}_check_import -* Tue Oct 26 2021 Tomas Orsava - 3.9-18 +* Wed Oct 20 2021 Tomas Orsava - 3.10-11 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix -* Wed Jul 07 2021 Miro Hrončok - 3.9-17 +* Tue Oct 12 2021 Lumír Balhar - 3.10-10 +- Non-existing path in py_reproducible_pyc_path causes build to fail +Resolves: rhbz#2011056 + +* Thu Sep 09 2021 Miro Hrončok - 3.10-9 +- Set $RPM_BUILD_ROOT in %%{python3_...} macros + to allow selecting alternate sysconfig install scheme based on that variable + +* Thu Sep 09 2021 Petr Viktorin - 3.10-8 +- Use --hardlink-dupes in %%py_byte_compile and brp-python-bytecompile + (for Python 3) +- Resolves: rhbz#1977895 + +* Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild + +* Wed Jul 07 2021 Miro Hrončok - 3.10-6 +- Move Python related BuildRoot Policy scripts from redhat-rpm-config to python-srpm-macros + +* Wed Jul 07 2021 Miro Hrončok - 3.10-5 - Introduce %%py3_check_import -* Mon Jun 28 2021 Miro Hrončok - 3.9-16 +* Wed Jun 30 2021 Miro Hrončok - 3.10-4 +- Include brp-python-hardlink in python-srpm-macros since it is no longer in RPM 4.17+ + +* Mon Jun 28 2021 Miro Hrončok - 3.10-3 - %%pytest: Set $PYTEST_ADDOPTS when %%{__pytest_addopts} is defined - Related: rhzb#1935212 -* Mon Mar 29 2021 Miro Hrončok - 3.9-15 +* Tue Jun 15 2021 Miro Hrončok - 3.10-2 +- Fix %%python_provide when fed python3.10-foo to obsolete python-foo instead of python--foo + +* Tue Jun 01 2021 Miro Hrončok - 3.10-1 +- Update main Python to Python 3.10 +- https://fedoraproject.org/wiki/Changes/Python3.10 + +* Tue Apr 27 2021 Miro Hrončok - 3.9-38 +- Escape %% symbols in macro files comments +- Fixes: rhbz#1953910 + +* Wed Apr 07 2021 Karolina Surma - 3.9-37 +- Use sysconfig.get_path() to get %%python3_sitelib and %%python3_sitearch +- Fixes: rhbz#1946972 + +* Mon Mar 29 2021 Miro Hrončok - 3.9-36 - Allow commas as argument separator for extras names in %%python_extras_subpkg - Fixes: rhbz#1936486 -* Sat Feb 20 2021 Miro Hrončok - 3.9-14 +* Sat Feb 20 2021 Miro Hrončok - 3.9-35 - Fix %%python_extras_subpkg with underscores in extras names +* Mon Feb 08 2021 Miro Hrončok - 3.9-34 +- Remove python2-rpm-macros +- https://fedoraproject.org/wiki/Changes/Disable_Python_2_Dist_RPM_Generators_and_Freeze_Python_2_Macros + * Fri Feb 05 2021 Miro Hrončok - 3.9-13 - Automatically word-wrap the description of extras subpackages - Fixes: rhbz#1922442 diff --git a/python.lua b/python.lua index 8766c91..bd30a85 100644 --- a/python.lua +++ b/python.lua @@ -2,22 +2,25 @@ -- Determine alternate names provided from the given name. -- Used in pythonname provides generator, python_provide and py_provides. --- There are 2 rules: --- python3-foo -> python-foo, python3X-foo --- python3X-foo -> python-foo, python3-foo +-- If only_3_to_3_X is false/nil/unused there are 2 rules: +-- python3-foo -> python-foo, python3.X-foo +-- python3.X-foo -> python-foo, python3-foo +-- If only_3_to_3_X is true there is only 1 rule: +-- python3-foo -> python3.X-foo -- There is no python-foo -> rule, python-foo packages are version agnostic. -- Returns a table/array with strings. Empty when no rule matched. -local function python_altnames(name) +local function python_altnames(name, only_3_to_3_X) local xy = rpm.expand('%{__default_python3_pkgversion}') local altnames = {} local replaced -- NB: dash needs to be escaped! if name:match('^python3%-') then - for i, prefix in ipairs({'python-', 'python' .. xy .. '-'}) do + local prefixes = only_3_to_3_X and {} or {'python-'} + for i, prefix in ipairs({'python' .. xy .. '-', table.unpack(prefixes)}) do replaced = name:gsub('^python3%-', prefix) table.insert(altnames, replaced) end - elseif name:match('^python' .. xy .. '%-') then + elseif name:match('^python' .. xy .. '%-') and not only_3_to_3_X then for i, prefix in ipairs({'python-', 'python3-'}) do replaced = name:gsub('^python' .. xy .. '%-', prefix) table.insert(altnames, replaced) @@ -27,42 +30,72 @@ local function python_altnames(name) end +local function __python_alttags(name, evr, tag_type) + -- for the "provides" tag_type we want also unversioned provides + local only_3_to_3_X = tag_type ~= "provides" + local operator = tag_type == "provides" and ' = ' or ' < ' + + -- global cache that tells what package NEVRs were already processed for the + -- given tag type + if __python_alttags_beenthere == nil then + __python_alttags_beenthere = {} + end + if __python_alttags_beenthere[tag_type] == nil then + __python_alttags_beenthere[tag_type] = {} + end + __python_alttags_beenthere[tag_type][name .. ' ' .. evr] = true + local alttags = {} + for i, altname in ipairs(python_altnames(name, only_3_to_3_X)) do + table.insert(alttags, altname .. operator .. evr) + end + return alttags +end + -- For any given name and epoch-version-release, return provides except self. -- Uses python_altnames under the hood -- Returns a table/array with strings. local function python_altprovides(name, evr) - -- global cache that tells what provides were already processed - if __python_altnames_provides_beenthere == nil then - __python_altnames_provides_beenthere = {} - end - __python_altnames_provides_beenthere[name .. ' ' .. evr] = true - local altprovides = {} - for i, altname in ipairs(python_altnames(name)) do - table.insert(altprovides, altname .. ' = ' .. evr) - end - return altprovides + return __python_alttags(name, evr, "provides") end +-- For any given name and epoch-version-release, return versioned obsoletes except self. +-- Uses python_altnames under the hood +-- Returns a table/array with strings. +local function python_altobsoletes(name, evr) + return __python_alttags(name, evr, "obsoletes") +end + + +local function __python_alttags_once(name, evr, tag_type) + -- global cache that tells what provides were already processed + if __python_alttags_beenthere == nil + or __python_alttags_beenthere[tag_type] == nil + or __python_alttags_beenthere[tag_type][name .. ' ' .. evr] == nil then + return __python_alttags(name, evr, tag_type) + else + return nil + end +end -- Like python_altprovides but only return something once. -- For each argument can only be used once, returns nil otherwise. -- Previous usage of python_altprovides counts as well. local function python_altprovides_once(name, evr) - -- global cache that tells what provides were already processed - if __python_altnames_provides_beenthere == nil then - __python_altnames_provides_beenthere = {} - end - if __python_altnames_provides_beenthere[name .. ' ' .. evr] == nil then - __python_altnames_provides_beenthere[name .. ' ' .. evr] = true - return python_altprovides(name, evr) - else - return nil - end + return __python_alttags_once(name, evr, "provides") +end + +-- Like python_altobsoletes but only return something once. +-- For each argument can only be used once, returns nil otherwise. +-- Previous usage of python_altobsoletes counts as well. +local function python_altobsoletes_once(name, evr) + return __python_alttags_once(name, evr, "obsoletes") end return { python_altnames = python_altnames, python_altprovides = python_altprovides, + python_altobsoletes = python_altobsoletes, python_altprovides_once = python_altprovides_once, + python_altobsoletes_once = python_altobsoletes_once, } diff --git a/rpminspect.yaml b/rpminspect.yaml new file mode 100644 index 0000000..4589e70 --- /dev/null +++ b/rpminspect.yaml @@ -0,0 +1,7 @@ +# completely disabled inspections: +inspections: + # there is no upstream and the files are changed from time to time + addedfiles: off + changedfiles: off + filesize: off + upstream: off diff --git a/tests/pythontest.spec b/tests/pythontest.spec index a0f052c..26eaa2c 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,11 +1,16 @@ %global basedir /opt/test/byte_compilation +# We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 +# Test with a representative of each, except 2.7 which we no longer have +%global python36_sitelib /usr/lib/python3.6/site-packages + Name: pythontest Version: 0 -Release: 0 +Release: 0%{?dist} Summary: ... License: MIT BuildRequires: python3-devel +BuildRequires: python3.6 %description ... @@ -19,15 +24,48 @@ echo "print()" > %{buildroot}%{basedir}/directory/to/test/recursion/file_in_dir. %py_byte_compile %{python3} %{buildroot}%{basedir}/file.py %py_byte_compile %{python3} %{buildroot}%{basedir}/directory +# Files in sitelib are compiled automatically by brp-python-bytecompile +mkdir -p %{buildroot}%{python3_sitelib}/directory/ +echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py + +mkdir -p %{buildroot}%{python36_sitelib}/directory/ +echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py + %check +LOCATIONS=" + %{buildroot}%{basedir} + %{buildroot}%{python3_sitelib}/directory/ + %{buildroot}%{python36_sitelib}/directory/ +" + # Count .py and .pyc files -PY=$(find %{buildroot}%{basedir} -name "*.py" | wc -l) -PYC=$(find %{buildroot}%{basedir} -name "*.pyc" | wc -l) +PY=$(find $LOCATIONS -name "*.py" | wc -l) +PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) + +# We should have 4 .py files (3 for python3, one for 3.6) +test $PY -eq 4 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files test $(expr $PY \* 2) -eq $PYC +# In this case the .pyc files should be identical across omtimization levels +# (they don't use docstrings and assert staements) +# So they should be hardlinked; the number of distinct inodes should match the +# number of source files. (Or be smaller, if the dupe detection is done +# across all files.) + +INODES=$(stat --format %i $(find $LOCATIONS -name "*.py[co]") | sort -u | wc -l) +test $PY -ge $INODES + + %files %pycached %{basedir}/file.py %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py +%pycached %{python3_sitelib}/directory/file.py +%pycached %{python36_sitelib}/directory/file.py + + +%changelog +* Thu Jan 01 2015 Fedora Packager - 0-0 +- This changelog entry exists and is deliberately set in the past diff --git a/tests/test_evals.py b/tests/test_evals.py index 2991328..e6cc8f7 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -16,11 +16,14 @@ XY = f'{sys.version_info[0]}{sys.version_info[1]}' # You can use * if you escape it from your Shell: # TESTED_FILES='macros.*' pytest -v # Remember that some tests might need more macros files than just -# the local ones. +# the local ones. You might need to use: +# TESTED_FILES='/usr/lib/rpm/macros:/usr/lib/rpm/platform/x86_64-linux/macros:macros.*' TESTED_FILES = os.getenv("TESTED_FILES", None) def rpm_eval(expression, fails=False, **kwargs): + if isinstance(expression, str): + expression = [expression] cmd = ['rpmbuild'] if TESTED_FILES: cmd += ['--macros', TESTED_FILES] @@ -29,7 +32,8 @@ def rpm_eval(expression, fails=False, **kwargs): cmd += ['--undefine', var] else: cmd += ['--define', f'{var} {value}'] - cmd += ['--eval', expression] + for e in expression: + cmd += ['--eval', e] cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if fails: @@ -50,6 +54,45 @@ def lib(): return lib_eval +def get_alt_x_y(): + """ + Some tests require alternate Python version to be installed. + In order to allow any Python version (or none at all), + this function/fixture exists. + You can control the behavior by setting the $ALTERNATE_PYTHON_VERSION + environment variable to X.Y (e.g. 3.6) or SKIP. + The environment variable must be set. + """ + env_name = "ALTERNATE_PYTHON_VERSION" + alternate_python_version = os.getenv(env_name, "") + if alternate_python_version.upper() == "SKIP": + pytest.skip(f"${env_name} set to SKIP") + if not alternate_python_version: + raise ValueError(f"${env_name} must be set, " + f"set it to SKIP if you want to skip tests that " + f"require alternate Python version.") + if not re.match(r"^\d+\.\d+$", alternate_python_version): + raise ValueError(f"${env_name} must be X.Y") + return alternate_python_version + + +def get_alt_xy(): + """ + Same as get_alt_x_y() but without a dot + """ + return get_alt_x_y().replace(".", "") + + +# We don't use decorators, to be able to call the functions directly +alt_x_y = pytest.fixture(scope="session")(get_alt_x_y) +alt_xy = pytest.fixture(scope="session")(get_alt_xy) + + +# https://fedoraproject.org/wiki/Changes/PythonSafePath +def safe_path_flag(x_y): + return 'P' if tuple(int(i) for i in x_y.split('.')) >= (3, 11) else '' + + def shell_stdout(script): return subprocess.check_output(script, env={**os.environ, 'LANG': 'C.utf-8'}, @@ -57,6 +100,17 @@ def shell_stdout(script): shell=True).rstrip() +@pytest.mark.parametrize('macro', ['%__python3', '%python3']) +def test_python3(macro): + assert rpm_eval(macro) == ['/usr/bin/python3'] + + +@pytest.mark.parametrize('macro', ['%__python3', '%python3']) +@pytest.mark.parametrize('pkgversion', ['3', '3.9', '3.12']) +def test_python3_with_pkgversion(macro, pkgversion): + assert rpm_eval(macro, python3_pkgversion=pkgversion) == [f'/usr/bin/python{pkgversion}'] + + @pytest.mark.parametrize('argument, result', [ ('a', 'a'), ('a-a', 'a-a'), @@ -81,8 +135,8 @@ def test_py3_dist(): assert rpm_eval(f'%py3_dist Aha[Boom] a') == ['python3dist(aha[boom]) python3dist(a)'] -def test_py3_dist_with_python3_pkgversion_redefined(): - assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion="3.6") == ['python3.6dist(aha[boom]) python3.6dist(a)'] +def test_py3_dist_with_python3_pkgversion_redefined(alt_x_y): + assert rpm_eval(f'%py3_dist Aha[Boom] a', python3_pkgversion=alt_x_y) == [f'python{alt_x_y}dist(aha[boom]) python{alt_x_y}dist(a)'] def test_python_provide_python(): @@ -131,67 +185,102 @@ def test_python_provide_doubleuse(): assert len(set(lines)) == 3 -def test_py_provides_python(): - lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 10]) +def test_py_provides_python(rhel): + lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python-foo = 6-1.fc66' in lines assert len(lines) == 1 -def test_py_provides_whatever(): - lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 12]) +def test_py_provides_whatever(rhel): + lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66', rhel=rhel) assert 'Provides: whatever = 6-1.fc66' in lines assert len(lines) == 1 -def test_py_provides_python3(): - lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 9]) +def test_py_provides_python3(rhel): + lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 + + +@pytest.mark.parametrize('rhel', [None, 9]) +def test_py_provides_python3_with_isa(rhel): + lines = rpm_eval('%py_provides python3-foo(x86_64)', version='6', release='1.fc66', rhel=rhel) + assert 'Provides: python3-foo(x86_64) = 6-1.fc66' in lines + assert 'Provides: python-foo(x86_64) = 6-1.fc66' in lines + assert f'Provides: python{X_Y}-foo(x86_64) = 6-1.fc66' in lines + assert f'Obsoletes: python{X_Y}-foo(x86_64) < 6-1.fc66' not in lines assert len(lines) == 3 -def test_py_provides_python3_epoched(): - lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 13]) +def test_py_provides_python3_epoched(rhel): + lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines - assert len(lines) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 1:6-1.fc66' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 -def test_py_provides_python3X(): - lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 13]) +def test_py_provides_python3X(rhel): + lines = rpm_eval(f'%py_provides python{X_Y}-foo', version='6', release='1.fc66', rhel=rhel) assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert 'Provides: python3-foo = 6-1.fc66' in lines assert len(lines) == 3 -def test_py_provides_python3X_epoched(): - lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66') +@pytest.mark.parametrize('rhel', [None, 27]) +def test_py_provides_python3X_epoched(rhel): + lines = rpm_eval(f'%py_provides python{X_Y}-foo', epoch='1', version='6', release='1.fc66', rhel=rhel) assert f'Provides: python{X_Y}-foo = 1:6-1.fc66' in lines assert 'Provides: python-foo = 1:6-1.fc66' in lines assert 'Provides: python3-foo = 1:6-1.fc66' in lines assert len(lines) == 3 -def test_py_provides_doubleuse(): +@pytest.mark.parametrize('rhel', [None, 2]) +def test_py_provides_doubleuse(rhel): lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}', - version='6', release='1.fc66') + version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 6-1.fc66' in lines assert 'Provides: python-foo = 6-1.fc66' in lines assert f'Provides: python{X_Y}-foo = 6-1.fc66' in lines - assert len(lines) == 6 - assert len(set(lines)) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 6-1.fc66' in lines + assert len(lines) == 8 + assert len(set(lines)) == 4 + else: + assert len(lines) == 6 + assert len(set(lines)) == 3 -def test_py_provides_with_evr(): +@pytest.mark.parametrize('rhel', [None, 2]) +def test_py_provides_with_evr(rhel): lines = rpm_eval('%py_provides python3-foo 123', - version='6', release='1.fc66') + version='6', release='1.fc66', rhel=rhel) assert 'Provides: python3-foo = 123' in lines assert 'Provides: python-foo = 123' in lines assert f'Provides: python{X_Y}-foo = 123' in lines - assert len(lines) == 3 + if rhel: + assert f'Obsoletes: python{X_Y}-foo < 123' in lines + assert len(lines) == 4 + else: + assert len(lines) == 3 def test_python_wheel_pkg_prefix(): @@ -225,8 +314,17 @@ def test_pytest_different_command(): def test_pytest_command_suffix(): lines = rpm_eval('%pytest -v') assert '/usr/bin/pytest -v' in lines[-1] - lines = rpm_eval('%pytest -v', python3_pkgversion="3.6", python3_version="3.6") - assert '/usr/bin/pytest-3.6 -v' in lines[-1] + +# this test does not require alternate Pythons to be installed +@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12']) +def test_pytest_command_suffix_alternate_pkgversion(version): + lines = rpm_eval('%pytest -v', python3_pkgversion=version, python3_version=version) + assert f'/usr/bin/pytest-{version} -v' in lines[-1] + + +def test_pytest_sets_pytest_xdist_auto_num_workers(): + lines = rpm_eval('%pytest', _smp_build_ncpus=2) + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-2}"' in '\n'.join(lines) def test_pytest_undefined_addopts_are_not_set(): @@ -262,6 +360,28 @@ def test_pytest_addopts_preserves_envvar(__pytest_addopts): assert 'z--' not in echoed +@pytest.mark.parametrize('__pytest_addopts', ['-X', None]) +def test_py3_test_envvars(lib, __pytest_addopts): + lines = rpm_eval('%{py3_test_envvars}\\\n%{python3} -m unittest', + buildroot='BUILDROOT', + _smp_build_ncpus='3', + __pytest_addopts=__pytest_addopts) + assert all(l.endswith('\\') for l in lines[:-1]) + stripped_lines = [l.strip(' \\') for l in lines] + sitearch = f'BUILDROOT/usr/{lib}/python{X_Y}/site-packages' + sitelib = f'BUILDROOT/usr/lib/python{X_Y}/site-packages' + assert f'PYTHONPATH="${{PYTHONPATH:-{sitearch}:{sitelib}}}"' in stripped_lines + assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines + assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines + assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines + assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-3}"' in stripped_lines + if __pytest_addopts: + assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines + else: + assert 'PYTEST_ADDOPTS' not in ''.join(lines) + assert stripped_lines[-1] == '/usr/bin/python3 -m unittest' + + def test_pypi_source_default_name(): urls = rpm_eval('%pypi_source', name='foo', version='6') @@ -318,13 +438,13 @@ def test_pypi_source_explicit_tilde(): def test_py3_shebang_fix(): cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/python3 -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' def test_py3_shebang_fix_default_shebang_flags(): lines = rpm_eval('%py3_shebang_fix arg1 arg2') lines[-1] = 'echo $shebang_flags' - assert shell_stdout('\n'.join(lines)) == '-kas' + assert shell_stdout('\n'.join(lines)) == f'-kas{safe_path_flag(X_Y)}' def test_py3_shebang_fix_custom_shebang_flags(): @@ -333,6 +453,31 @@ def test_py3_shebang_fix_custom_shebang_flags(): assert shell_stdout('\n'.join(lines)) == '-kaEs' +@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s) + lines[-1] = 'echo $shebang_flags' + expected = f'-ka{safe_path_flag(X_Y)}' if safe_path_flag(X_Y) else '-k' + assert shell_stdout('\n'.join(lines)) == expected + + +@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_P(_py3_shebang_P): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_P=_py3_shebang_P) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-kas' + + +@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}']) +@pytest.mark.parametrize('_py3_shebang_P', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_sP(_py3_shebang_s, _py3_shebang_P): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', + _py3_shebang_s=_py3_shebang_s, + _py3_shebang_P=_py3_shebang_P) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-k' + + @pytest.mark.parametrize('flags', [None, '%{nil}']) def test_py3_shebang_fix_no_shebang_flags(flags): lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags) @@ -342,7 +487,7 @@ def test_py3_shebang_fix_no_shebang_flags(flags): def test_py_shebang_fix_custom_python(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/pypy -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' def test_pycached_in_sitelib(): @@ -361,11 +506,14 @@ def test_pycached_in_sitearch(lib): ] -def test_pycached_in_36(): - lines = rpm_eval('%pycached /usr/lib/python3.6/site-packages/foo*.py') +# this test does not require alternate Pythons to be installed +@pytest.mark.parametrize('version', ['3.6', '3.7', '3.12']) +def test_pycached_with_alternate_version(version): + version_nodot = version.replace('.', '') + lines = rpm_eval(f'%pycached /usr/lib/python{version}/site-packages/foo*.py') assert lines == [ - '/usr/lib/python3.6/site-packages/foo*.py', - '/usr/lib/python3.6/site-packages/__pycache__/foo*.cpython-36{,.opt-?}.pyc' + f'/usr/lib/python{version}/site-packages/foo*.py', + f'/usr/lib/python{version}/site-packages/__pycache__/foo*.cpython-{version_nodot}{{,.opt-?}}.pyc' ] @@ -403,7 +551,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -414,7 +562,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected @@ -471,6 +619,60 @@ def test_python_extras_subpkg_F(): assert lines == expected +def test_python_extras_subpkg_a(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + BuildArch: noarch + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_A(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 6-7 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + +def test_python_extras_subpkg_aA(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -A -F toml', + version='6', release='7', fails=True) + assert lines[0] == ('error: %python_extras_subpkg: simultaneous -a ' + '(insert BuildArch: noarch) and -A (do not insert ' + 'BuildArch: noarch (default)) options are not possible') + + +def test_python_extras_subpkg_v(): + lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', + version='6', release='7') + expected = textwrap.dedent(f""" + %package -n python3-setuptools_scm+toml + Summary: Metapackage for python3-setuptools_scm: toml extras + Requires: python3-setuptools_scm = 1.2.3 + %description -n python3-setuptools_scm+toml + This is a metapackage bringing in toml extras requires for + python3-setuptools_scm. + It makes sure the dependencies are installed. + """).lstrip().splitlines() + assert lines == expected + + def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') @@ -559,28 +761,43 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_platform', '%python_platform_triplet', '%python_ext_suffix', + '%python_cache_tag', '%py_shebang_fix', '%py_build', - '%py_build_egg', '%py_build_wheel', '%py_install', - '%py_install_egg', '%py_install_wheel', '%py_check_import', + '%py_test_envvars', ]) @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) - assert lines == ['error: attempt to use unversioned python, ' - 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'] + # strip the deprecation message + if 'deprecated' in lines[0]: + lines = lines[1:] + assert lines[0] == ( + 'error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' + ) + # when the macros are %global, the error is longer + # we deliberately allow this extra line to be optional + if len(lines) > 1 and "error: lua script failed" not in lines[1]: + # the failed macro is not unnecessarily our tested macro + pattern = r'error: Macro %\S+ failed to expand' + assert re.match(pattern, lines[1]) + # but there should be no more lines + assert len(lines) < 3 @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) + unverisoned = rpm_eval(macro, __python='/usr/bin/python3') + expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] + assert unverisoned == expected # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -597,30 +814,72 @@ def test_ext_suffix(): assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"] -def test_python_sitelib_value(): +def test_cache_tag(): + assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"] + + +def test_cache_tag_alternate_python(alt_x_y, alt_xy): + assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"] + + +def test_cache_tag_alternate_python3(alt_x_y, alt_xy): + assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"] + + +def test_python_sitelib_value_python3(): macro = '%python_sitelib' - assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages'] -def test_python3_sitelib_value(): +def test_python_sitelib_value_alternate_python(alt_x_y): + macro = '%python_sitelib' + assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/lib/python{alt_x_y}/site-packages'] + + +def test_python3_sitelib_value_default(): macro = '%python3_sitelib' - assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] assert rpm_eval(macro) == [f'/usr/lib/python{X_Y}/site-packages'] -def test_python_sitearch_value(lib): +def test_python3_sitelib_value_alternate_python(alt_x_y): + macro = '%python3_sitelib' + assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == + rpm_eval(macro, python3_pkgversion=alt_x_y) == + [f'/usr/lib/python{alt_x_y}/site-packages']) + + +def test_python3_sitelib_value_alternate_prefix(): + macro = '%python3_sitelib' + assert rpm_eval(macro, _prefix='/app') == [f'/app/lib/python{X_Y}/site-packages'] + + +def test_python_sitearch_value_python3(lib): macro = '%python_sitearch' - assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] assert rpm_eval(macro, __python='%__python3') == [f'/usr/{lib}/python{X_Y}/site-packages'] -def test_python3_sitearch_value(lib): +def test_python_sitearch_value_alternate_python(lib, alt_x_y): + macro = '%python_sitearch' + assert rpm_eval(macro, __python=f'/usr/bin/python{alt_x_y}') == [f'/usr/{lib}/python{alt_x_y}/site-packages'] + + +def test_python3_sitearch_value_default(lib): macro = '%python3_sitearch' - assert rpm_eval(macro, __python3='/usr/bin/python3.6') == [f'/usr/{lib}/python3.6/site-packages'] assert rpm_eval(macro) == [f'/usr/{lib}/python{X_Y}/site-packages'] +def test_python3_sitearch_value_alternate_python(lib, alt_x_y): + macro = '%python3_sitearch' + assert (rpm_eval(macro, __python3=f'/usr/bin/python{alt_x_y}') == + rpm_eval(macro, python3_pkgversion=alt_x_y) == + [f'/usr/{lib}/python{alt_x_y}/site-packages']) + + +def test_python3_sitearch_value_alternate_prefix(lib): + macro = '%python3_sitearch' + assert rpm_eval(macro, _prefix='/app') == [f'/app/{lib}/python{X_Y}/site-packages'] + + @pytest.mark.parametrize( 'args, expected_args', [ @@ -636,20 +895,20 @@ def test_python3_sitearch_value(lib): @pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', - '/usr/bin/python3.6']) + '/usr/bin/pythonX.Y']) def test_py3_check_import(args, expected_args, __python3, lib): x_y = X_Y macros = { 'buildroot': 'BUILDROOT', '_rpmconfigdir': 'RPMCONFIGDIR', - 'py3_shebang_flags': 's', } if __python3 is not None: + if 'X.Y' in __python3: + __python3 = __python3.replace('X.Y', get_alt_x_y()) macros['__python3'] = __python3 # If the __python3 command has version at the end, parse it and expect it. # Note that the command is used to determine %python3_sitelib and %python3_sitearch, # so we only test known CPython schemes here and not PyPy for simplicity. - # We also only test main Python + 3.6 because those are required by the CI config. if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) @@ -666,7 +925,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" _PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} + {__python3 or '/usr/bin/python3'} -s{safe_path_flag(x_y)} RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) assert lines == expected.splitlines() @@ -674,6 +933,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): @pytest.mark.parametrize( 'shebang_flags_value, expected_shebang_flags', [ + ('sP', '-sP'), ('s', '-s'), ('%{nil}', ''), (None, ''), @@ -690,3 +950,58 @@ def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_s # Compare the last line of the command, that's where lua part is evaluated expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' assert lines[-1].strip() == expected + + +def test_multi_python(alt_x_y): + """ + Ensure memoized %python_version works when switching %__python back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python /usr/bin/python{version}', '%python_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +def test_multi_python3(alt_x_y): + """ + Ensure memoized %python3_version works when switching %__python3 back + and forth. + """ + versions = ['3', alt_x_y, X_Y, '3'] + evals = [] + for version in versions: + evals.extend((f'%global __python3 /usr/bin/python{version}', '%python3_version')) + lines = rpm_eval(evals) + lines = [l for l in lines if l] # strip empty lines generated by %global + assert lines == [X_Y, alt_x_y, X_Y, X_Y] + + +@pytest.mark.parametrize('macro', [ + '%py3_build', + '%py3_build_wheel', + '%py3_install', +]) +def test_deprecation(macro): + lines = rpm_eval(macro) + assert "is deprecated" in lines[0] + assert f"{macro} " in lines[0] + + +def test_multiple_deprecation(): + source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' + lines = rpm_eval(source) + + assert "is deprecated" in lines[0] + assert "%py3_build " in lines[0] + + assert "is deprecated" in lines[1] + assert "%py3_build_wheel " in lines[1] + + assert "is deprecated" in lines[2] + assert "%py3_install " in lines[2] + + assert "is deprecated" not in '\n'.join(lines[3:]) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 52e1d7e..dd10689 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['this_is_a_module_in_cwd']) @@ -141,15 +141,15 @@ def test_modules_from_sys_path_found(tmp_path): def test_modules_from_file_are_found(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' - test_file.write_text('math\nwave\nsunau\n') + test_file.write_text('math\nwave\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau'): + for m in ('math', 'wave', 'csv'): sys.modules.pop(m, None) modules_main(['-f', str(test_file)]) - assert 'sunau' in sys.modules + assert 'csv' in sys.modules assert 'math' in sys.modules assert 'wave' in sys.modules @@ -160,22 +160,22 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('sunau\npathlib\n') - test_file_3.write_text('logging\nsunau\n') + test_file_2.write_text('csv\nnetrc\n') + test_file_3.write_text('logging\ncsv\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): + for m in ('math', 'wave', 'csv', 'netrc', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): + for module in ('csv', 'math', 'wave', 'netrc', 'logging'): assert module in sys.modules def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,24 +253,70 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(ModuleNotFoundError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_module_with_error_propagates_exception(tmp_path): +def test_import_module_returns_failed_modules(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + assert failed_modules == ['this_is_a_module_in_tmp_path_2'] + + +def test_module_with_error_propagates_exception(tmp_path, capsys): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - # The correct exception must be raised - with pytest.raises(ZeroDivisionError): + with pytest.raises(SystemExit): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) + _, err = capsys.readouterr() + assert "ZeroDivisionError" in err + + +def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) + assert failed_modules == [] + + +def test_all_modules_are_imported(tmp_path, capsys): + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + with pytest.raises(SystemExit): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'missing_module', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ]) + _, err = capsys.readouterr() + for i in range(1, 4): + assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err + assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py new file mode 100644 index 0000000..7546dc0 --- /dev/null +++ b/tests/test_rpm_in_distinfo.py @@ -0,0 +1,41 @@ +from pathlib import Path + +import os +import pytest +import subprocess + +@pytest.fixture +def create_test_files(tmp_path): + def _create(subpath, installer_content): + dir_path = tmp_path / subpath + dir_path.mkdir(parents=True, exist_ok=True) + installer_file = dir_path / "INSTALLER" + installer_file.write_text(installer_content) + record_file = dir_path / "RECORD" + record_file.write_text("dummy content in RECORD file\n") + return dir_path + return _create + +testdata = [ + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), + ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), + ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), + ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), +] +@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) +def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, +path, installer_content, expected_installer_content, record_file_exists): + script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") + tmp_dir = create_test_files(path, installer_content) + monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) + result = subprocess.run( + [script_path], + capture_output=True, text=True + ) + + assert result.returncode == 0 + assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content + assert Path(tmp_dir / "RECORD").exists() is record_file_exists + diff --git a/tests/tests.yml b/tests/tests.yml deleted file mode 100644 index 28a9bac..0000000 --- a/tests/tests.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -- hosts: localhost - tags: - - classic - tasks: - - dnf: - name: "*" - state: latest - -- hosts: localhost - roles: - - role: standard-test-basic - tags: - - classic - tests: - - pytest: - dir: . - run: PYTHONPATH=/usr/lib/rpm/redhat pytest -v - - manual_byte_compilation: - dir: . - run: rpmbuild -ba pythontest.spec - required_packages: - - rpm-build - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-pytest - - python3.6 - diff --git a/tests/testwheel.spec b/tests/testwheel.spec new file mode 100644 index 0000000..461d7f3 --- /dev/null +++ b/tests/testwheel.spec @@ -0,0 +1,116 @@ +Name: testwheel +Epoch: 42 +Version: 1 +Release: 0%{?dist} +Summary: ... +License: MIT +BuildArch: noarch +BuildRequires: python3-devel +BuildRequires: python3-setuptools >= 61 +BuildRequires: python3-pip + +%description +This builds and installs a wheel which we can then use as a test for +%%python_wheel_inject_sbom. + + +%prep +cat > pyproject.toml << EOF +[project] +name = "testwheel" +version = "1" + +[build-system] +requires = ["setuptools >= 61"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["testwheel*"] +EOF +# create a secondary dist-info folder in the project +# we need to ensure this file is not altered +mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info +touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +echo 'recursive-include testwheel/_vendor *' > MANIFEST.in + + +%build +export PIP_CONFIG_FILE=/dev/null +%{python3} -m pip wheel . --no-build-isolation + +# The macro should happily alter multiple wheels, let's make more +for i in {1..5}; do + mkdir ${i} + cp -a *.whl ${i} +done + +# using relative paths should succeed +%python_wheel_inject_sbom {1..5}/*.whl + +# repetitive use should bail out and fail (SBOM is already there) +%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true + +# each wheel should already have it, all should fail individually as well +for i in {1..5}; do + %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true +done + + +%install +mkdir -p %{buildroot}%{python_wheel_dir} +cp -a *.whl %{buildroot}%{python_wheel_dir} + +# using absolute paths should work +%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl + +# and fail when repeated +%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true + + +%check +%define venvsite venv/lib/python%{python3_version}/site-packages +%{python3} -m venv venv +venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl + +test -f %{venvsite}/testwheel-1.dist-info/RECORD +test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD +# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) +sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' + +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD +test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true + +# this deliberately uses a different mechanism than the macro +# if you are running this test on a different distro, adjust it +%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} + +PYTHONOPTIMIZE=0 %{python3} -c " +import json +with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: + sbom = json.load(fp) +assert len(sbom['components']) == 1 +assert sbom['components'][0]['type'] == 'library' +assert sbom['components'][0]['name'] == 'testwheel' +assert sbom['components'][0]['version'] == '1-0%{?dist}' +assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' +" + +# replace the installation with the original unaltered wheel +venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl +test -f %{venvsite}/testwheel-1.dist-info/RECORD +# no SBOM +test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json +grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true + + +%files +%{python_wheel_dir}/*.whl + + +%changelog +* Wed Aug 13 2025 Miro Hrončok - 42:1-0 +- A static changelog with a date, so we can clamp mtimes