diff --git a/.fmf/version b/.fmf/version deleted file mode 100644 index d00491f..0000000 --- a/.fmf/version +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/brp-fix-pyc-reproducibility b/brp-fix-pyc-reproducibility index 05857b3..536a126 100644 --- a/brp-fix-pyc-reproducibility +++ b/brp-fix-pyc-reproducibility @@ -1,18 +1,20 @@ -#!/bin/bash -eu +#!/bin/bash -e # If using normal root, avoid changing anything. -if [[ -z "${RPM_BUILD_ROOT:-}" ]] || [[ "${RPM_BUILD_ROOT:-}" = "/" ]]; then +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi # Defined as %py_reproducible_pyc_path macro and passed here as # the first command-line argument -path_to_fix=${1:?} +path_to_fix=$1 # First, check that the parser is available: -if [[ ! -x /usr/bin/marshalparser ]]; then +if [ ! -x /usr/bin/marshalparser ]; then echo "ERROR: If %py_reproducible_pyc_path is defined, you have to also BuildRequire: /usr/bin/marshalparser !" exit 1 fi -find "$path_to_fix" -type f -name '*.pyc' -exec /usr/bin/marshalparser --fix --overwrite '{}' '+' +# Set pipefail so if $path_to_fix does not exist, the build fails +set -o pipefail +find "$path_to_fix" -type f -name "*.pyc" | xargs /usr/bin/marshalparser --fix --overwrite diff --git a/brp-python-bytecompile b/brp-python-bytecompile index 1911fa1..c1749ab 100644 --- a/brp-python-bytecompile +++ b/brp-python-bytecompile @@ -6,28 +6,16 @@ errors_terminate=$2 # Therefore $1 ($default_python) is not needed and is invoked with "" by default. # $default_python stays in the arguments for backward compatibility and $extra for the following check: extra=$3 -if [[ 0"$extra" -eq 1 ]]; then +if [ 0$extra -eq 1 ]; then echo -e "%_python_bytecompile_extra is discontinued, use %py_byte_compile instead.\nSee: https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3" >/dev/stderr exit 1 fi -compileall_flags="$4" - # If using normal root, avoid changing anything. -if [[ -z "$RPM_BUILD_ROOT" ]] || [[ "$RPM_BUILD_ROOT" = "/" ]]; then +if [ -z "$RPM_BUILD_ROOT" -o "$RPM_BUILD_ROOT" = "/" ]; then exit 0 fi -# This function clamps the source mtime, see https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes -function python_clamp_source_mtime() -{ - local _=$1 - local python_binary=$2 - local _=$3 - local python_libdir="$4" - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m clamp_source_mtime -q "$python_libdir" -} - # This function now implements Python byte-compilation in three different ways: # Python >= 3.4 and < 3.9 uses a new module compileall2 - https://github.com/fedora-python/compileall2 # In Python >= 3.9, compileall2 was merged back to standard library (compileall) so we can use it directly again. @@ -36,23 +24,22 @@ function python_bytecompile() { local options=$1 local python_binary=$2 - # local exclude=$3 # No longer used + local exclude=$3 local python_libdir="$4" - local compileall_flags="$5" python_version=$($python_binary -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") # # Python 3.4 and higher # - if [[ "$python_version" -ge 34 ]]; then + if [ "$python_version" -ge 34 ]; then # We compile all opt levels in one go: only when $options is empty. - if [[ -n "$options" ]]; then + if [ -n "$options" ]; then return fi - if [[ "$python_version" -ge 39 ]]; then + if [ "$python_version" -ge 39 ]; then # For Pyhon 3.9+, use the standard library compileall_module=compileall else @@ -60,23 +47,18 @@ function python_bytecompile() compileall_module=compileall2 fi - if [[ "$python_version" -ge 37 ]]; then - # Force the TIMESTAMP invalidation mode - invalidation_option=--invalidation-mode=timestamp - else - # For older Pythons, the option does not exist - # as the invalidation is always based on size+mtime - invalidation_option= - fi + [ ! -z $exclude ] && exclude="-x '$exclude'" # PYTHONPATH is needed for compileall2, but doesn't hurt for the stdlib # -o 0 -o 1 are the optimization levels # -q disables verbose output # -f forces the process to overwrite existing compiled files + # -x excludes paths defined by regex # -e excludes symbolic links pointing outside the build root + # -x and -e together implements the same functionality as the Filter class below # -s strips $RPM_BUILD_ROOT from the path # -p prepends the leading slash to the path to make it absolute - PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module $compileall_flags -o 0 -o 1 -q -f -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes $invalidation_option -e "$RPM_BUILD_ROOT" "$python_libdir" + PYTHONPATH=/usr/lib/rpm/redhat/ $python_binary -B -m $compileall_module -o 0 -o 1 -q -f $exclude -s "$RPM_BUILD_ROOT" -p / --hardlink-dupes -e "$RPM_BUILD_ROOT" "$python_libdir" else # @@ -92,10 +74,13 @@ python_libdir = "$python_libdir" depth = sys.getrecursionlimit() real_libdir = "$real_libdir" build_root = "$RPM_BUILD_ROOT" +exclude = r"$exclude" class Filter: def search(self, path): ret = not os.path.realpath(path).startswith(build_root) + if exclude: + ret = ret or re.search(exclude, path) return ret sys.exit(not compileall.compile_dir(python_libdir, depth, real_libdir, force=1, rx=Filter(), quiet=1)) @@ -129,21 +114,16 @@ do echo "Bytecompiling .py files below $python_libdir using $python_binary" # Generate normal (.pyc) byte-compiled files. - python_clamp_source_mtime "" "$python_binary" "" "$python_libdir" "" - if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then - # One or more of the files had inaccessible mtime - exit 1 - fi - python_bytecompile "" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then + python_bytecompile "" "$python_binary" "" "$python_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi # Generate optimized (.pyo) byte-compiled files. # N.B. For Python 3.4+, this call does nothing - python_bytecompile "-O" "$python_binary" "" "$python_libdir" "$compileall_flags" - if [[ $? -ne 0 ]] && [[ 0"$errors_terminate" -ne 0 ]]; then + python_bytecompile "-O" "$python_binary" "" "$python_libdir" + if [ $? -ne 0 -a 0$errors_terminate -ne 0 ]; then # One or more of the files had a syntax error exit 1 fi diff --git a/brp-python-rpm-in-distinfo b/brp-python-rpm-in-distinfo deleted file mode 100755 index b72d704..0000000 --- a/brp-python-rpm-in-distinfo +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/bash - -set -eu -# If using normal root, avoid changing anything. -if [[ "${RPM_BUILD_ROOT:-/}" = "/" ]] ; then - exit 0 -fi - -find "$RPM_BUILD_ROOT" -name 'INSTALLER' -type f -print0|grep -z -E "/usr/lib(64)?/python3\.[0-9]+/site-packages/[^/]+\.dist-info/INSTALLER" | while read -d "" installer ; do - if cmp -s <(echo pip) "$installer" ; then - echo "rpm" > "$installer" - rm -f "$(dirname "$installer")/RECORD" - fi -done -exit 0 diff --git a/clamp_source_mtime.py b/clamp_source_mtime.py deleted file mode 100644 index 1d03a6b..0000000 --- a/clamp_source_mtime.py +++ /dev/null @@ -1,163 +0,0 @@ -"""Module/script to clamp the mtimes of all .py files to $SOURCE_DATE_EPOCH - -When called as a script with arguments, this compiles the directories -given as arguments recursively. - -If upstream is interested, this can be later integrated to the compileall module -as an additional option (e.g. --clamp-source-mtime). - -License: -This has been derived from the Python's compileall module -and it follows Python licensing. For more info see: https://www.python.org/psf/license/ -""" -from __future__ import print_function -import os -import sys - -# Python 3.6 and higher -PY36 = sys.version_info[0:2] >= (3, 6) - -__all__ = ["clamp_dir", "clamp_file"] - - -def _walk_dir(dir, maxlevels, quiet=0): - if PY36 and quiet < 2 and isinstance(dir, os.PathLike): - dir = os.fspath(dir) - else: - dir = str(dir) - if not quiet: - print('Listing {!r}...'.format(dir)) - try: - names = os.listdir(dir) - except OSError: - if quiet < 2: - print("Can't list {!r}".format(dir)) - names = [] - names.sort() - for name in names: - if name == '__pycache__': - continue - fullname = os.path.join(dir, name) - if not os.path.isdir(fullname): - yield fullname - elif (maxlevels > 0 and name != os.curdir and name != os.pardir and - os.path.isdir(fullname) and not os.path.islink(fullname)): - for result in _walk_dir(fullname, maxlevels=maxlevels - 1, - quiet=quiet): - yield result - - -def clamp_dir(dir, source_date_epoch, quiet=0): - """Clamp the mtime of all modules in the given directory tree. - - Arguments: - - dir: the directory to byte-compile - source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH - quiet: full output with False or 0, errors only with 1, - no output with 2 - """ - maxlevels = sys.getrecursionlimit() - files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) - success = True - for file in files: - if not clamp_file(file, source_date_epoch, quiet=quiet): - success = False - return success - - -def clamp_file(fullname, source_date_epoch, quiet=0): - """Clamp the mtime of one file. - - Arguments: - - fullname: the file to byte-compile - source_date_epoch: integer parsed from $SOURCE_DATE_EPOCH - quiet: full output with False or 0, errors only with 1, - no output with 2 - """ - if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): - fullname = os.fspath(fullname) - else: - fullname = str(fullname) - name = os.path.basename(fullname) - - if os.path.isfile(fullname) and not os.path.islink(fullname): - if name[-3:] == '.py': - try: - mtime = int(os.stat(fullname).st_mtime) - atime = int(os.stat(fullname).st_atime) - except OSError as e: - if quiet >= 2: - return False - elif quiet: - print('*** Error checking mtime of {!r}...'.format(fullname)) - else: - print('*** ', end='') - print(e.__class__.__name__ + ':', e) - return False - if mtime > source_date_epoch: - if not quiet: - print('Clamping mtime of {!r}'.format(fullname)) - try: - os.utime(fullname, (atime, source_date_epoch)) - except OSError as e: - if quiet >= 2: - return False - elif quiet: - print('*** Error clamping mtime of {!r}...'.format(fullname)) - else: - print('*** ', end='') - print(e.__class__.__name__ + ':', e) - return False - return True - - -def main(): - """Script main program.""" - import argparse - - source_date_epoch = os.getenv('SOURCE_DATE_EPOCH') - if not source_date_epoch: - print("Not clamping source mtimes, $SOURCE_DATE_EPOCH not set") - return True # This is a success, no action needed - try: - source_date_epoch = int(source_date_epoch) - except ValueError: - print("$SOURCE_DATE_EPOCH must be an integer") - return False - - parser = argparse.ArgumentParser( - description='Clamp .py source mtime to $SOURCE_DATE_EPOCH.') - parser.add_argument('-q', action='count', dest='quiet', default=0, - help='output only error messages; -qq will suppress ' - 'the error messages as well.') - parser.add_argument('clamp_dest', metavar='FILE|DIR', nargs='+', - help=('zero or more file and directory paths ' - 'to clamp')) - - args = parser.parse_args() - clamp_dests = args.clamp_dest - - success = True - try: - for dest in clamp_dests: - if os.path.isfile(dest): - if not clamp_file(dest, quiet=args.quiet, - source_date_epoch=source_date_epoch): - success = False - else: - if not clamp_dir(dest, quiet=args.quiet, - source_date_epoch=source_date_epoch): - success = False - return success - except KeyboardInterrupt: - if args.quiet < 2: - print("\n[interrupted]") - return False - return True - - -if __name__ == '__main__': - exit_status = int(not main()) - sys.exit(exit_status) diff --git a/compileall2.py b/compileall2.py index ea7e76f..c58e545 100644 --- a/compileall2.py +++ b/compileall2.py @@ -4,7 +4,7 @@ When called as a script with arguments, this compiles the directories given as arguments recursively; the -l option prevents it from recursing into directories. -Without arguments, it compiles all modules on sys.path, without +Without arguments, if compiles all modules on sys.path, without recursing into subdirectories. (Even though it should do so for packages -- for now, you'll have to deal with packages separately.) @@ -36,11 +36,11 @@ PY35 = sys.version_info[0:2] >= (3, 5) # introduced in Python 3.7. These cases are covered by variables here or by PY37 # variable itself. if PY37: - pyc_struct_format = '<4sLL' + pyc_struct_format = '<4sll' pyc_header_lenght = 12 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) else: - pyc_struct_format = '<4sL' + pyc_struct_format = '<4sl' pyc_header_lenght = 8 pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) @@ -106,7 +106,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, workers: maximum number of parallel workers invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beginning of original file path, applied + prependdir: path to prepend to beggining of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path @@ -120,34 +120,23 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, stripdir = dir prependdir = ddir ddir = None - if workers < 0: - raise ValueError('workers must be greater or equal to 0') - if workers != 1: - # Check if this is a system where ProcessPoolExecutor can function. - from concurrent.futures.process import _check_system_limits - try: - _check_system_limits() - except NotImplementedError: - workers = 1 - else: - from concurrent.futures import ProcessPoolExecutor + if workers is not None: + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + elif workers != 1: + try: + # Only import when needed, as low resource platforms may + # fail to import it + from concurrent.futures import ProcessPoolExecutor + except ImportError: + workers = 1 if maxlevels is None: maxlevels = sys.getrecursionlimit() files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) success = True - if workers != 1 and ProcessPoolExecutor is not None: - mp_context_arg = {} - if PY37: - import multiprocessing - if multiprocessing.get_start_method() == 'fork': - mp_context = multiprocessing.get_context('forkserver') - else: - mp_context = None - mp_context_arg = {"mp_context": mp_context} - # If workers == 0, let ProcessPoolExecutor choose + if workers is not None and workers != 1 and ProcessPoolExecutor is not None: workers = workers or None - with ProcessPoolExecutor(max_workers=workers, - **mp_context_arg) as executor: + with ProcessPoolExecutor(max_workers=workers) as executor: results = executor.map(partial(compile_file, ddir=ddir, force=force, rx=rx, quiet=quiet, @@ -189,7 +178,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, files each with one optimization level. invalidation_mode: how the up-to-dateness of the pyc will be checked stripdir: part of path to left-strip from source file path - prependdir: path to prepend to beginning of original file path, applied + prependdir: path to prepend to beggining of original file path, applied after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. @@ -201,8 +190,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, "in combination with stripdir or prependdir")) success = True - fullname = os.fspath(fullname) - stripdir = os.fspath(stripdir) if stripdir is not None else None + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) name = os.path.basename(fullname) dfile = None @@ -215,13 +206,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if stripdir is not None: fullname_parts = fullname.split(os.path.sep) stripdir_parts = stripdir.split(os.path.sep) + ddir_parts = list(fullname_parts) - if stripdir_parts != fullname_parts[:len(stripdir_parts)]: - if quiet < 2: - print("The stripdir path {!r} is not a valid prefix for " - "source path {!r}; ignoring".format(stripdir, fullname)) - else: - dfile = os.path.join(*fullname_parts[len(stripdir_parts):]) + for spart, opart in zip(stripdir_parts, fullname_parts): + if spart == opart: + ddir_parts.remove(spart) + + dfile = os.path.join(*ddir_parts) if prependdir is not None: if dfile is None: @@ -267,7 +258,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not force: try: mtime = int(os.stat(fullname).st_mtime) - expect = struct.pack(*(pyc_header_format + (mtime & 0xFFFF_FFFF,))) + expect = struct.pack(*(pyc_header_format + (mtime,))) for cfile in opt_cfiles.values(): with open(cfile, 'rb') as chandle: actual = chandle.read(pyc_header_lenght) @@ -310,8 +301,9 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: print('*** ', end='') # escape non-printable characters in msg - encoding = sys.stdout.encoding or sys.getdefaultencoding() - msg = err.msg.encode(encoding, errors='backslashreplace').decode(encoding) + msg = err.msg.encode(sys.stdout.encoding, + errors='backslashreplace') + msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False @@ -416,8 +408,8 @@ def main(): type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', help=('Optimization levels to run compilation with. ' - 'Default is -1 which uses the optimization level ' - 'of the Python interpreter itself (see -O).')) + 'Default is -1 which uses optimization level of ' + 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') parser.add_argument('--hardlink-dupes', action='store_true', @@ -464,8 +456,7 @@ def main(): # if flist is provided then load it if args.flist: try: - with (sys.stdin if args.flist=='-' else - open(args.flist, encoding="utf-8")) as f: + with (sys.stdin if args.flist=='-' else open(args.flist)) as f: for line in f: compile_dests.append(line.strip()) except OSError: @@ -473,6 +464,9 @@ def main(): print("Error reading file list {}".format(args.flist)) return False + if args.workers is not None: + args.workers = args.workers or None + if PY37 and args.invalidation_mode: ivl_mode = args.invalidation_mode.replace('-', '_').upper() invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] diff --git a/import_all_modules.py b/import_all_modules.py index 97e924c..3930236 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -7,7 +7,6 @@ import os import re import site import sys -import traceback from contextlib import contextmanager from pathlib import Path @@ -94,24 +93,11 @@ def read_modules_from_all_args(args): def import_modules(modules): '''Procedure to perform import check for each module name from the given list of modules. - - Return a list of failed modules. ''' - failed_modules = [] - for module in modules: print('Check import:', module, file=sys.stderr) - try: - importlib.import_module(module) - except Exception: - traceback.print_exc(file=sys.stderr) - failed_modules.append(module) - - if failed_modules: - print(f'Failed to import: {", ".join(failed_modules)}', file=sys.stderr) - - return failed_modules + importlib.import_module(module) def argparser(): @@ -178,10 +164,7 @@ def main(argv=None): with remove_unwanteds_from_sys_path(): addsitedirs_from_environ() - failed_modules = import_modules(modules) - - if failed_modules: - raise SystemExit(1) + import_modules(modules) if __name__ == '__main__': diff --git a/macros.pybytecompile b/macros.pybytecompile index e81ddae..dd8b495 100644 --- a/macros.pybytecompile +++ b/macros.pybytecompile @@ -17,14 +17,8 @@ # Python 3.11+ no longer needs this: https://github.com/python/cpython/pull/27926 (but we support older Pythons as well) %py_byte_compile()\ -clamp_source_mtime () {\ - python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} %1"\ - bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m clamp_source_mtime $bytecode_compilation_path \ -}\ -\ py2_byte_compile () {\ - python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ failure=0\ find $bytecode_compilation_path -type f -a -name "*.py" -print0 | xargs -0 $python_binary -s -c 'import py_compile, sys; [py_compile.compile(f, dfile=f.partition("'"$RPM_BUILD_ROOT"'")[2], doraise=True) for f in sys.argv[1:]]' || failure=1\ @@ -32,38 +26,28 @@ py2_byte_compile () {\ test $failure -eq 0\ }\ \ -py34_byte_compile () {\ - python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ +py3_byte_compile () {\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ -}\ -py37_byte_compile () {\ - python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ - bytecode_compilation_path="%2"\ - PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + PYTHONPATH="%{_rpmconfigdir}/redhat" $python_binary -s -B -m compileall2 -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ py39_byte_compile () {\ - python_binary="%{__env_unset_source_date_epoch_if_not_clamp_mtime} PYTHONHASHSEED=0 %1"\ + python_binary="env PYTHONHASHSEED=0 %1"\ bytecode_compilation_path="%2"\ - $python_binary -s -B -m compileall %{?_smp_build_ncpus:-j%{_smp_build_ncpus}} -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes --invalidation-mode=timestamp $bytecode_compilation_path \ + $python_binary -s -B -m compileall -o 0 -o 1 -s $RPM_BUILD_ROOT -p / --hardlink-dupes $bytecode_compilation_path \ }\ \ # Path to intepreter should not contain any arguments \ [[ "%1" =~ " -" ]] && echo "ERROR py_byte_compile: Path to interpreter should not contain any arguments" >&2 && exit 1 \ -# First, clamp source mtime https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes \ -clamp_source_mtime "%1" "%2"; \ # Get version without a dot (36 instead of 3.6), bash doesn't compare floats well \ python_version=$(%1 -c "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") \ # compileall2 is an enhanced fork of stdlib compileall module for Python >= 3.4 \ # and it was merged back to stdlib in Python >= 3.9 \ -# Only Python 3.7+ supports and needs the --invalidation-mode option \ if [ "$python_version" -ge 39 ]; then \ py39_byte_compile "%1" "%2"; \ -elif [ "$python_version" -ge 37 ]; then \ -py37_byte_compile "%1" "%2"; \ elif [ "$python_version" -ge 34 ]; then \ -py34_byte_compile "%1" "%2"; \ +py3_byte_compile "%1" "%2"; \ else \ py2_byte_compile "%1" "%2"; \ fi diff --git a/macros.python b/macros.python index 4bd0da6..c73e21d 100644 --- a/macros.python +++ b/macros.python @@ -1,78 +1,20 @@ -# Memoize a macro to avoid calling the same expensive code multiple times in -# the specfile. -# There is no error handling, -# memoizing an undefined macro (or using such a key) has undefined behavior. -# Options: -# -n - The name of the macro to wrap -# -k - The name of the macro to use as a cache key -%_python_memoize(n:k:) %{lua: -local name = opt.n --- NB: We use rpm.expand() here instead of the macros table to make sure errors --- are propogated properly. -local cache_key = rpm.expand("%{" .. opt.k .. "}") -if not _python_macro_cache then - -- This is intentionally a global lua table - _python_macro_cache = {} -end -if not _python_macro_cache[cache_key] then - _python_macro_cache[cache_key] = {} -end -if not _python_macro_cache[cache_key][name] then - _python_macro_cache[cache_key][name] = rpm.expand("%{" .. name .. "}") -end -print(_python_macro_cache[cache_key][name]) -} - -# Deprecation wrapper, warns only once per macro -# Options: -# -n - The name of the macro that is deprecated -%_python_deprecated(n:) %{lua: -if not _python_deprecated_warned then - -- This is intentionally a global lua table - _python_deprecated_warned = {} -end -if not _python_deprecated_warned[opt.n] then - _python_deprecated_warned[opt.n] = true - local msg = "The %" .. opt.n .. " macro is deprecated and will likely stop working in Fedora 44. " .. - "See the current Python packaging guidelines: " .. - "https://docs.fedoraproject.org/en-US/packaging-guidelines/Python/" - macros.warn({msg}) -end -} - # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%__python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_sitelib %{_python_memoize -n __python_sitelib -k __python} - -%__python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python_sitearch %{_python_memoize -n __python_sitearch -k __python} - -%__python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version %{_python_memoize -n __python_version -k __python} - -%__python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_version_nodots %{_python_memoize -n __python_version_nodots -k __python} - -%__python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform %{_python_memoize -n __python_platform -k __python} - -%__python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_platform_triplet %{_python_memoize -n __python_platform_triplet -k __python} - -%__python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python_ext_suffix %{_python_memoize -n __python_ext_suffix -k __python} - -%__python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") -%python_cache_tag %{_python_memoize -n __python_cache_tag -k __python} +%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py %_py_shebang_s s -%__py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") -%_py_shebang_P %{_python_memoize -n __py_shebang_P -k __python} +%_py_shebang_P %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") %py_shbang_opts -%{?_py_shebang_s}%{?_py_shebang_P} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) @@ -86,22 +28,33 @@ end # Use the slashes after expand so that the command starts on the same line as # the macro -%py_build() %{_python_deprecated -n py_build}%{expand:\\\ +%py_build() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} build --executable="%{__python} %{py_shbang_opts}" %{?*} } -%py_build_wheel() %{_python_deprecated -n py_build_wheel}%{expand:\\\ +%py_build_egg() %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python} %{py_setup} %{?py_setup_args} bdist_egg %{?*} +} + +%py_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py_install() %{_python_deprecated -n py_install}%{expand:\\\ +%py_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } +%py_install_egg() %{expand:\\\ + mkdir -p %{buildroot}%{python_sitelib} + %{__python} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python_version}.egg %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ +} + %py_install_wheel() %{expand:\\\ %{__python} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ @@ -185,8 +138,7 @@ end PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}} %python_disable_dependency_generator() \ %undefine __pythondist_requires \ diff --git a/macros.python-srpm b/macros.python-srpm index d5cd4e5..7104f7f 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -17,7 +17,7 @@ # There are two macros: # # This always contains the major.minor version (with dots), default for %%python3_version. -%__default_python3_version 3.14 +%__default_python3_version 3.11 # # The pkgname version that determines the alternative provide name (e.g. python3.9-foo), # set to the same as above, but historically hasn't included the dot. @@ -66,8 +66,6 @@ ### BRP scripts (and related macros) -## Modifies installation method in .dist-info/INSTALLER file to rpm -%python_rpm_in_distinfo 1 ## Automatically compile python files %py_auto_byte_compile 1 ## Should python bytecompilation errors terminate a build? @@ -75,24 +73,18 @@ ## Should python bytecompilation compile outside python specific directories? ## This always causes errors when enabled, see https://fedoraproject.org/wiki/Changes/No_more_automagic_Python_bytecompilation_phase_3 %_python_bytecompile_extra 0 -## Helper macro to unset $SOURCE_DATE_EPOCH if %%clamp_mtime_to_source_date_epoch is not set -## https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes#Python_bytecode -%__env_unset_source_date_epoch_if_not_clamp_mtime %[0%{?clamp_mtime_to_source_date_epoch} == 0 ? "env -u SOURCE_DATE_EPOCH" : "env"] ## The individual BRP scripts -%__brp_python_rpm_in_distinfo %{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo -%__brp_python_bytecompile %{__env_unset_source_date_epoch_if_not_clamp_mtime} %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" "%{?_smp_build_ncpus:-j%{_smp_build_ncpus}}" +%__brp_python_bytecompile %{_rpmconfigdir}/redhat/brp-python-bytecompile "" "%{?_python_bytecompile_errors_terminate_build}" "%{?_python_bytecompile_extra}" %__brp_fix_pyc_reproducibility %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility %__brp_python_hardlink %{_rpmconfigdir}/redhat/brp-python-hardlink ## This macro is included in redhat-rpm-config's %%__os_install_post # Note that the order matters: -# 1. brp-python-rpm-in-distinfo modifies .dist-info/INSTALLER file -# 2. brp-python-bytecompile can create (or replace) pyc files -# 3. brp-fix-pyc-reproducibility can modify the pyc files from above -# 4. brp-python-hardlink de-duplicates identical pyc files +# 1. brp-python-bytecompile can create (or replace) pyc files +# 2. brp-fix-pyc-reproducibility can modify the pyc files from above +# 3. brp-python-hardlink de-duplicates identical pyc files %__os_install_post_python \ - %{?python_rpm_in_distinfo:%{?__brp_python_rpm_in_distinfo}} \ %{?py_auto_byte_compile:%{?__brp_python_bytecompile}} \ %{?py_reproducible_pyc_path:%{?__brp_fix_pyc_reproducibility} "%{py_reproducible_pyc_path}"} \ %{?__brp_python_hardlink} \ @@ -230,20 +222,15 @@ end } -%python_extras_subpkg(n:i:f:FaAv:) %{expand:%{lua: +%python_extras_subpkg(n:i:f:F) %{expand:%{lua: local option_n = '-n (name of the base package)' local option_i = '-i (buildroot path to metadata)' local option_f = '-f (builddir path to a filelist)' local option_F = '-F (skip %%files section)' - local option_a = '-a (insert BuildArch: noarch)' - local option_A = '-A (do not insert BuildArch: noarch (default))' local value_n = rpm.expand('%{-n*}') local value_i = rpm.expand('%{-i*}') local value_f = rpm.expand('%{-f*}') local value_F = rpm.expand('%{-F}') - local value_a = rpm.expand('%{-a}') - local value_A = rpm.expand('%{-A}') - local value_v = rpm.expand('%{-v}') local args = rpm.expand('%{*}') if value_n == '' then rpm.expand('%{error:%%%0: missing option ' .. option_n .. '}') @@ -260,14 +247,10 @@ if value_f ~= '' and value_F ~= '' then rpm.expand('%{error:%%%0: simultaneous ' .. option_f .. ' and ' .. option_F .. ' options are not possible}') end - if value_a ~= '' and value_A ~= '' then - rpm.expand('%{error:%%%0: simultaneous ' .. option_a .. ' and ' .. option_A .. ' options are not possible}') - end if args == '' then rpm.expand('%{error:%%%0 requires at least one argument with "extras" name}') end - local verrel = rpm.expand('%{?-v*}%{!?-v:%{version}-%{release}}') - local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}' .. verrel + local requires = 'Requires: ' .. value_n .. ' = %{?epoch:%{epoch}:}%{version}-%{release}' for extras in args:gmatch('[^%s,]+') do local rpmname = value_n .. '+' .. extras local pkgdef = '%package -n ' .. rpmname @@ -287,15 +270,11 @@ 'It makes sure the dependencies are installed.\\\n' local files = '' if value_i ~= '' then - files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost %dir ' .. value_i + files = '%files -n ' .. rpmname .. '\\\n' .. '%ghost ' .. value_i elseif value_f ~= '' then files = '%files -n ' .. rpmname .. ' -f ' .. value_f end - local tags = summary .. '\\\n' .. requires - if value_a ~= '' then - tags = tags .. '\\\nBuildArch: noarch' - end - for i, line in ipairs({pkgdef, tags, description, files, ''}) do + for i, line in ipairs({pkgdef, summary, requires, description, files, ''}) do print(line .. '\\\n') end end diff --git a/macros.python-wheel-sbom b/macros.python-wheel-sbom deleted file mode 100644 index 41389f9..0000000 --- a/macros.python-wheel-sbom +++ /dev/null @@ -1,124 +0,0 @@ -# The macros in this file are used to add SBOM to wheel files that we ship. -# Majority of Python packages will not need to do that, -# as they only use wheels as an intermediate artifact. -# The macros will be used by packages installing wheel to %%python_wheel_dir -# or by Python interpreters bundling their own (patched) wheels. -# -# The runtime dependencies are not Required by the python-rpm-macros package, -# users of this macro need to specify them on their own or rely on the fact that -# they are all available in the default buildroot. -# -# Usage: %%python_wheel_inject_sbom PATHS_TO_WHEELS -# -# The wheels are modified in-place. - - -# Path of the SBOM file in the PEP 770 .dist-info/sboms directory -# This filename is explicitly mentioned in https://cyclonedx.org/specification/overview/ -# section Recognized file patterns -%__python_wheel_sbom_filename bom.json - - -# The SBOM content to put to the file -# This is a CycloneDX component as recommended in https://discuss.python.org/t/97436/7 -%__python_wheel_sbom_content %{expand:{ - "bomFormat": "CycloneDX", - "specVersion": "1.6", - "components": [ - { - "type": "library", - "name": "%{name}", - "version": "%{version}-%{release}", - "purl": "%{__python_wheel_purl}" - } - ] -}} - - -# The purl used above -# We use the src package name (which is easier to get and more useful to consumers). -# Note that epoch needs special handling, see https://github.com/package-url/purl-spec/issues/69 -# and https://redhatproductsecurity.github.io/security-data-guidelines/purl/ -%__python_wheel_purl pkg:rpm/%{__python_wheel_dist_purl_namespace}/%{name}@%{version}-%{release}?%{?epoch:epoch=%{epoch}&}arch=src - - -# The purl namespace used above -# https://lists.fedoraproject.org/archives/list/packaging@lists.fedoraproject.org/thread/GTRCTAF3R3SSBVEJYFCATKNRT7RYVFQI/ -# Distributors, define %%dist_purl_namespace to set this. -# The rest of the code is fallback for distributions without it (relying on %%dist_name). -%__python_wheel_dist_purl_namespace %{?dist_purl_namespace}%{!?dist_purl_namespace:%{lua: - if macros.epel then - -- being epel beats the %%dist_name value - -- added in https://src.fedoraproject.org/rpms/epel-rpm-macros/pull-request/86 - print("epel") - else - local dist_map = { - -- fedora is in the purl-spec examples https://github.com/package-url/purl-spec/blob/main/PURL-TYPES.rst#rpm - -- added in https://src.fedoraproject.org/rpms/fedora-release/pull-request/385 - ["Fedora Linux"] = "fedora", - -- added in https://gitlab.com/redhat/centos-stream/rpms/centos-stream-release/-/merge_requests/7 - ["CentOS Stream"] = "centos", - -- documented at https://redhatproductsecurity.github.io/security-data-guidelines/purl/ - ["Red Hat Enterprise Linux"] = "redhat", - -- documented at https://wiki.almalinux.org/documentation/sbom-guide.html - ["AlmaLinux"] = "almalinux", - -- from https://github.com/google/osv.dev/pull/2939 - ["Rocky Linux"] = "rocky-linux", - } - print(dist_map[macros.dist_name] or "unknown") - end -}} - - -# A Bash scriptlet to inject the SBOM file into the wheel(s) -# The macro takes positional nargs+ with wheel paths -# For each wheel, it -# 1. aborts if the SBOM file is already there (it won't override) -# 2. inserts the SBOM file to .dist-info/sboms -# 3. amends .dist-info/RECORD with the added SBOM file -%python_wheel_inject_sbom() %{expand:( - %[%# ? "" : "%{error:%%%0: At least one argument (wheel path) is required}"] - - set -eu -o pipefail - export LANG=C.utf-8 - - tmpdir=$(mktemp -d) - trap 'rm -rf "$tmpdir"' EXIT - pwd0=$(pwd) - ret=0 - - for whl in %{*}; do - cd "$tmpdir" - if [[ "$whl" != /* ]]; then - whl="$pwd0/$whl" - fi - - record=$(zipinfo -1 "$whl" | grep -E '^[^/]+-[^/]+\.dist-info/RECORD$') - distinfo="${record%%/RECORD}" - bom="$distinfo/sboms/%{__python_wheel_sbom_filename}" - - if zipinfo -1 "$whl" | grep -qFx "$bom"; then - echo -e "\\n\\nERROR %%%%%0: $whl already has $bom, aborting\\n\\n" >&2 - ret=1 - continue - fi - - unzip "$whl" "$record" - mkdir "$distinfo/sboms" - echo '%{__python_wheel_sbom_content}' > "$bom" - checksum="sha256=$(sha256sum "$bom" | cut -f1 -d' ')" - size="$(wc --bytes "$bom" | cut -f1 -d' ')" - echo "$bom,$checksum,$size" >> "$record" - - if [[ -n "${SOURCE_DATE_EPOCH:-}" ]]; then - touch --date="@$SOURCE_DATE_EPOCH" "$bom" "$record" - fi - - zip -r "$whl" "$record" "$bom" - rm -rf "$distinfo" - cd "$pwd0" - done - - exit $ret -)} - diff --git a/macros.python3 b/macros.python3 index 38cdcd4..204b0de 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,35 +1,18 @@ # nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) # so we set it manually (to empty string), making our Python prefer the correct install scheme location # platbase/base is explicitly set to %%{_prefix} to support custom values, such as /app for flatpaks -%__python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_sitelib %{_python_memoize -n __python3_sitelib -k __python3} - -%__python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") -%python3_sitearch %{_python_memoize -n __python3_sitearch -k __python3} - -%__python3_version %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version %{_python_memoize -n __python3_version -k __python3} - -%__python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_version_nodots %{_python_memoize -n __python3_version_nodots -k __python3} - -%__python3_platform %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python3_platform %{_python_memoize -n __python3_platform -k __python3} - -%__python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_platform_triplet %{_python_memoize -n __python3_platform_triplet -k __python3} - -%__python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") -%python3_ext_suffix %{_python_memoize -n __python3_ext_suffix -k __python3} - -%__python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Esc "import sys; print(sys.implementation.cache_tag)") -%python3_cache_tag %{_python_memoize -n __python3_cache_tag -k __python3} - +%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib', vars={'platbase': '%{_prefix}', 'base': '%{_prefix}'}))") +%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %_py3_shebang_s s -%__py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") -%_py3_shebang_P %{_python_memoize -n __py3_shebang_P -k __python3} +%_py3_shebang_P %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print('P' if hasattr(sys.flags, 'safe_path') else '')") %py3_shbang_opts -%{?_py3_shebang_s}%{?_py3_shebang_P} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) @@ -43,22 +26,33 @@ # Use the slashes after expand so that the command starts on the same line as # the macro -%py3_build() %{_python_deprecated -n py3_build}%{expand:\\\ +%py3_build() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} build --executable="%{__python3} %{py3_shbang_opts}" %{?*} } -%py3_build_wheel() %{_python_deprecated -n py3_build_wheel}%{expand:\\\ +%py3_build_egg() %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + %{__python3} %{py_setup} %{?py_setup_args} bdist_egg %{?*} +} + +%py3_build_wheel() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} bdist_wheel %{?*} } -%py3_install() %{_python_deprecated -n py3_install}%{expand:\\\ +%py3_install() %{expand:\\\ CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ %{__python3} %{py_setup} %{?py_setup_args} install -O1 --skip-build --root %{buildroot} --prefix %{_prefix} %{?*} rm -rfv %{buildroot}%{_bindir}/__pycache__ } +%py3_install_egg() %{expand:\\\ + mkdir -p %{buildroot}%{python3_sitelib} + %{__python3} -m easy_install -m --prefix %{buildroot}%{_prefix} -Z dist/*-py%{python3_version}.egg %{?*} + rm -rfv %{buildroot}%{_bindir}/__pycache__ +} + %py3_install_wheel() %{expand:\\\ %{__python3} -m pip install -I dist/%{1} --root %{buildroot} --prefix %{_prefix} --no-deps --no-index --no-warn-script-location rm -rfv %{buildroot}%{_bindir}/__pycache__ @@ -118,8 +112,7 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}\\\ - PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-%{_smp_build_ncpus}}"} + %{?__pytest_addopts:PYTEST_ADDOPTS="${PYTEST_ADDOPTS:-} %{__pytest_addopts}"}} # This is intended for Python 3 only, hence also no Python version in the name. %__pytest /usr/bin/pytest%(test %{python3_pkgversion} == 3 || echo -%{python3_version}) diff --git a/pathfix.py b/pathfix.py index 6808382..1d7db3a 100644 --- a/pathfix.py +++ b/pathfix.py @@ -56,6 +56,7 @@ def main(): if recursedown(arg): bad = 1 elif os.path.islink(arg): err(arg + ': will not process symbolic links\n') + bad = 1 else: if fix(arg): bad = 1 sys.exit(bad) diff --git a/plan.fmf b/plan.fmf deleted file mode 100644 index 88850dd..0000000 --- a/plan.fmf +++ /dev/null @@ -1,40 +0,0 @@ -execute: - how: tmt - -discover: - - name: same_repo - how: shell - tests: - - name: pytest - test: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v - - name: manual_byte_compilation_clamp_mtime_off - path: /tests - test: rpmbuild --define 'dist .clamp0' --define 'clamp_mtime_to_source_date_epoch 0' -ba pythontest.spec - - name: manual_byte_compilation_clamp_mtime_on - path: /tests - test: rpmbuild --define 'dist .clamp1' --define 'clamp_mtime_to_source_date_epoch 1' -ba pythontest.spec - - name: rpmlint_clamp_mtime_off - test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp0.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - name: rpmlint_clamp_mtime_on - test: rpmlint ~/rpmbuild/RPMS/x86_64/pythontest-0-0.clamp1.x86_64.rpm | grep python-bytecode-inconsistent-mtime || exit 0 && exit 1 - - name: python_wheel_inject_sbom - path: /tests - test: rpmbuild -ba testwheel.spec - -prepare: - - name: Install dependencies - how: install - package: - - rpm-build - - rpmlint - - python-rpm-macros - - python3-rpm-macros - - python3-devel - - python3-setuptools - - python3-pip - - python3-pytest - - python3.6 - - dnf - - name: Update packages - how: shell - script: dnf upgrade -y diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 263853a..1726983 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -8,18 +8,16 @@ Source101: macros.python Source102: macros.python-srpm Source104: macros.python3 Source105: macros.pybytecompile -Source106: macros.python-wheel-sbom # Lua files Source201: python.lua # Python code -%global compileall2_version 0.8.0 +%global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py %global pathfix_version 1.0.0 Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py -Source304: clamp_source_mtime.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -34,15 +32,13 @@ Source402: brp-python-hardlink # This one is from redhat-rpm-config < 190 # It has no upstream yet Source403: brp-fix-pyc-reproducibility -# brp script to write "rpm" string into the .dist-info/INSTALLER file -Source404: brp-python-rpm-in-distinfo # macros and lua: MIT # import_all_modules.py: MIT -# compileall2.py, clamp_source_mtime.py: PSF-2.0 -# pathfix.py: PSF-2.0 -# brp scripts: GPL-2.0-or-later -License: MIT AND PSF-2.0 AND GPL-2.0-or-later +# compileall2.py: PSFv2 +# pathfix.py: PSFv2 +# brp scripts: GPLv2+ +License: MIT and Python and GPLv2+ # The package version MUST be always the same as %%{__default_python3_version}. # To have only one source of truth, we load the macro and use it. @@ -56,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 6%{?dist} BuildArch: noarch @@ -124,7 +120,6 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ -install -m 644 clamp_source_mtime.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -135,11 +130,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ # It also ensures that: # - our BRPs can execute # - if our BRPs affect this package, we don't need to build it twice -%define add_buildroot() %{lua:print((macros[macros[1]]:gsub(macros._rpmconfigdir, macros.buildroot .. macros._rpmconfigdir)))} -%global __brp_python_bytecompile %{add_buildroot __brp_python_bytecompile} -%global __brp_python_hardlink %{add_buildroot __brp_python_hardlink} -%global __brp_fix_pyc_reproducibility %{add_buildroot __brp_fix_pyc_reproducibility} -%global __brp_python_rpm_in_distinfo %{add_buildroot __brp_python_rpm_in_distinfo} +%global __brp_python_bytecompile %{buildroot}%{__brp_python_bytecompile} +%global __brp_python_hardlink %{buildroot}%{__brp_python_hardlink} +%global __brp_fix_pyc_reproducibility %{buildroot}%{__brp_fix_pyc_reproducibility} %check @@ -150,18 +143,15 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile -%{rpmmacrodir}/macros.python-wheel-sbom %{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/pathfix.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py -%{_rpmconfigdir}/redhat/clamp_source_mtime.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility -%{_rpmconfigdir}/redhat/brp-python-rpm-in-distinfo %{_rpmluadir}/fedora/srpm/python.lua %files -n python3-rpm-macros @@ -169,100 +159,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true %changelog -* Thu Oct 16 2025 Miro Hrončok - 3.14-9 -- %%python_extras_subpkg: Only %%ghost the egg-info/dist-info directory, not the content -- That way, accidentally unpackaged files within are reported as errors - -* Tue Sep 09 2025 Miro Hrončok - 3.14-8 -- %%python_extras_subpkg: Add -v option to specify the required version(-release) -- This is useful when the extras are built from a different specfile (e.g. in EPEL for a RHEL base package) - -* Fri Aug 29 2025 Miro Hrončok - 3.14-7 -- %%python_wheel_inject_sbom: Don't accidentally alter nested .dist-infos - -* Wed Aug 13 2025 Miro Hrončok - 3.14-6 -- Introduce %%python_wheel_inject_sbom - -* Mon Aug 11 2025 Lumír Balhar - 3.14-5 -- import_all_modules: Add error handling for import failures - -* Fri Jul 25 2025 Fedora Release Engineering -- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild - -* Mon Jul 21 2025 Íñigo Huguet - 3.14-3 -- pathfix.py: Don't fail on symbolic links - -* Sun Jun 29 2025 Miro Hrončok - 3.14-2 -- Deprecate %%py3_build, %%py3_build_wheel, and %%py3_install -- Deprecate %%py_build, %%py_build_wheel, and %%py_install -- https://fedoraproject.org/wiki/Changes/DeprecateSetuppyMacros - -* Wed May 28 2025 Karolina Surma - 3.14-1 -- Update main Python to 3.14 - -* Mon Feb 10 2025 Tomáš Hrnčiar - 3.13-5 -- Add brp script to modify .dist-info/INSTALLER file - -* Sat Jan 18 2025 Fedora Release Engineering - 3.13-4 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_42_Mass_Rebuild - -* Fri Jul 19 2024 Fedora Release Engineering - 3.12-3 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild - -* Tue Jun 25 2024 Cristian Le - 3.13-2 -- %%python_extras_subpkg: Add option -a to include BuildArch: noarch - -* Thu Jun 06 2024 Karolina Surma - 3.13-1 -- Update main Python to 3.13 - -* Thu Mar 28 2024 Zbigniew Jędrzejewski-Szmek - 3.12-9 -- Minor improvements to brp-fix-pyc-reproducibility - -* Fri Mar 22 2024 Lumír Balhar - 3.12-8 -- Update bundled compileall2 to version 0.8.0 - -* Thu Jan 25 2024 Miro Hrončok - 3.12-7 -- %%py3_test_envvars: Only set $PYTEST_XDIST_AUTO_NUM_WORKERS if not already set - -* Mon Jan 22 2024 Fedora Release Engineering - 3.12-6 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild - -* Mon Oct 09 2023 Maxwell G - 3.12-5 -- Fix python macro memoizing to account for changing %%__python3 - -* Tue Sep 05 2023 Maxwell G - 3.12-4 -- Remove %%py3_build_egg and %%py3_install_egg macros. - -* Wed Aug 09 2023 Karolina Surma - 3.12-3 -- Declare the license as an SPDX expression - -* Fri Jul 21 2023 Fedora Release Engineering - 3.12-2 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild - -* Tue Jun 13 2023 Tomáš Hrnčiar - 3.12-1 -- Update main Python to Python 3.12 -- https://fedoraproject.org/wiki/Changes/Python3.12 - -* Thu Mar 16 2023 Miro Hrončok - 3.11-10 -- Don't assume %%_smp_mflags only ever contains -jX, use -j%%_smp_build_ncpus directly -- Fixes: rhbz#2179149 - -* Fri Jan 20 2023 Miro Hrončok - 3.11-9 -- Memoize values of macros that execute python to get their value -- Fixes: rhbz#2155505 - -* Fri Jan 20 2023 Fedora Release Engineering - 3.11-8 -- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild - -* Mon Dec 19 2022 Miro Hrončok - 3.11-7 -- Bytecompilation: Unset $SOURCE_DATE_EPOCH when %%clamp_mtime_to_source_date_epoch is not set -- Bytecompilation: Pass --invalidation-mode=timestamp to compileall (on Python 3.7+) -- Bytecompilation: Clamp source mtime: https://fedoraproject.org/wiki/Changes/ReproducibleBuildsClampMtimes -- Bytecompilation: Compile Python files in parallel, according to %%_smp_mflags - * Sun Nov 13 2022 Miro Hrončok - 3.11-6 -- Set PYTEST_XDIST_AUTO_NUM_WORKERS=%%{_smp_build_ncpus} from %%pytest -- pytest-xdist 3+ respects this value when -n auto is used - Expose the environment variables used by %%pytest via %%{py3_test_envvars} * Tue Oct 25 2022 Lumír Balhar - 3.11-5 diff --git a/tests/pythontest.spec b/tests/pythontest.spec index 26eaa2c..3b707c5 100644 --- a/tests/pythontest.spec +++ b/tests/pythontest.spec @@ -1,16 +1,18 @@ %global basedir /opt/test/byte_compilation # We have 3 different ways of bytecompiling: for 3.9+, 3.4-3.8, and 2.7 -# Test with a representative of each, except 2.7 which we no longer have +# Test with a representative of each. %global python36_sitelib /usr/lib/python3.6/site-packages +%global python27_sitelib /usr/lib/python2.7/site-packages Name: pythontest Version: 0 -Release: 0%{?dist} +Release: 0 Summary: ... License: MIT BuildRequires: python3-devel BuildRequires: python3.6 +BuildRequires: python2.7 %description ... @@ -31,19 +33,23 @@ echo "print()" > %{buildroot}%{python3_sitelib}/directory/file.py mkdir -p %{buildroot}%{python36_sitelib}/directory/ echo "print()" > %{buildroot}%{python36_sitelib}/directory/file.py +mkdir -p %{buildroot}%{python27_sitelib}/directory/ +echo "print()" > %{buildroot}%{python27_sitelib}/directory/file.py + %check LOCATIONS=" %{buildroot}%{basedir} %{buildroot}%{python3_sitelib}/directory/ %{buildroot}%{python36_sitelib}/directory/ + %{buildroot}%{python27_sitelib}/directory/ " # Count .py and .pyc files PY=$(find $LOCATIONS -name "*.py" | wc -l) PYC=$(find $LOCATIONS -name "*.py[co]" | wc -l) -# We should have 4 .py files (3 for python3, one for 3.6) -test $PY -eq 4 +# We should have 5 .py files (3 for python3, one each for 3.6 & 2.7) +test $PY -eq 5 # Every .py file should be byte-compiled to two .pyc files (optimization level 0 and 1) # so we should have two times more .pyc files than .py files @@ -64,8 +70,4 @@ test $PY -ge $INODES %pycached %{basedir}/directory/to/test/recursion/file_in_dir.py %pycached %{python3_sitelib}/directory/file.py %pycached %{python36_sitelib}/directory/file.py - - -%changelog -* Thu Jan 01 2015 Fedora Packager - 0-0 -- This changelog entry exists and is deliberately set in the past +%{python27_sitelib}/directory/file.py* diff --git a/tests/test_evals.py b/tests/test_evals.py index e6cc8f7..8e5eb29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -22,8 +22,6 @@ TESTED_FILES = os.getenv("TESTED_FILES", None) def rpm_eval(expression, fails=False, **kwargs): - if isinstance(expression, str): - expression = [expression] cmd = ['rpmbuild'] if TESTED_FILES: cmd += ['--macros', TESTED_FILES] @@ -32,8 +30,7 @@ def rpm_eval(expression, fails=False, **kwargs): cmd += ['--undefine', var] else: cmd += ['--define', f'{var} {value}'] - for e in expression: - cmd += ['--eval', e] + cmd += ['--eval', expression] cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if fails: @@ -322,11 +319,6 @@ def test_pytest_command_suffix_alternate_pkgversion(version): assert f'/usr/bin/pytest-{version} -v' in lines[-1] -def test_pytest_sets_pytest_xdist_auto_num_workers(): - lines = rpm_eval('%pytest', _smp_build_ncpus=2) - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-2}"' in '\n'.join(lines) - - def test_pytest_undefined_addopts_are_not_set(): lines = rpm_eval('%pytest', __pytest_addopts=None) assert 'PYTEST_ADDOPTS' not in '\n'.join(lines) @@ -374,7 +366,6 @@ def test_py3_test_envvars(lib, __pytest_addopts): assert 'PATH="BUILDROOT/usr/bin:$PATH"' in stripped_lines assert 'CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"' in stripped_lines assert 'PYTHONDONTWRITEBYTECODE=1' in stripped_lines - assert 'PYTEST_XDIST_AUTO_NUM_WORKERS="${PYTEST_XDIST_AUTO_NUM_WORKERS:-3}"' in stripped_lines if __pytest_addopts: assert f'PYTEST_ADDOPTS="${{PYTEST_ADDOPTS:-}} {__pytest_addopts}"' in stripped_lines else: @@ -551,7 +542,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+toml - %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info %package -n python3-setuptools_scm+yaml Summary: Metapackage for python3-setuptools_scm: yaml extras @@ -562,7 +553,7 @@ def test_python_extras_subpkg_i(): It makes sure the dependencies are installed. %files -n python3-setuptools_scm+yaml - %ghost %dir /usr/lib/python{X_Y}/site-packages/*.egg-info + %ghost /usr/lib/python{X_Y}/site-packages/*.egg-info """).lstrip().splitlines() assert lines == expected @@ -619,60 +610,6 @@ def test_python_extras_subpkg_F(): assert lines == expected -def test_python_extras_subpkg_a(): - lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -F toml', - version='6', release='7') - expected = textwrap.dedent(f""" - %package -n python3-setuptools_scm+toml - Summary: Metapackage for python3-setuptools_scm: toml extras - Requires: python3-setuptools_scm = 6-7 - BuildArch: noarch - %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for - python3-setuptools_scm. - It makes sure the dependencies are installed. - """).lstrip().splitlines() - assert lines == expected - - -def test_python_extras_subpkg_A(): - lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -F toml', - version='6', release='7') - expected = textwrap.dedent(f""" - %package -n python3-setuptools_scm+toml - Summary: Metapackage for python3-setuptools_scm: toml extras - Requires: python3-setuptools_scm = 6-7 - %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for - python3-setuptools_scm. - It makes sure the dependencies are installed. - """).lstrip().splitlines() - assert lines == expected - - -def test_python_extras_subpkg_aA(): - lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -a -A -F toml', - version='6', release='7', fails=True) - assert lines[0] == ('error: %python_extras_subpkg: simultaneous -a ' - '(insert BuildArch: noarch) and -A (do not insert ' - 'BuildArch: noarch (default)) options are not possible') - - -def test_python_extras_subpkg_v(): - lines = rpm_eval('%python_extras_subpkg -n python3-setuptools_scm -A -v 1.2.3 -F toml', - version='6', release='7') - expected = textwrap.dedent(f""" - %package -n python3-setuptools_scm+toml - Summary: Metapackage for python3-setuptools_scm: toml extras - Requires: python3-setuptools_scm = 1.2.3 - %description -n python3-setuptools_scm+toml - This is a metapackage bringing in toml extras requires for - python3-setuptools_scm. - It makes sure the dependencies are installed. - """).lstrip().splitlines() - assert lines == expected - - def test_python_extras_subpkg_underscores(): lines = rpm_eval('%python_extras_subpkg -n python3-webscrapbook -F adhoc_ssl', version='0.33.3', release='1.fc33') @@ -764,8 +701,10 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_cache_tag', '%py_shebang_fix', '%py_build', + '%py_build_egg', '%py_build_wheel', '%py_install', + '%py_install_egg', '%py_install_wheel', '%py_check_import', '%py_test_envvars', @@ -775,29 +714,14 @@ unversioned_macros = pytest.mark.parametrize('macro', [ @unversioned_macros def test_unversioned_python_errors(macro): lines = rpm_eval(macro, fails=True) - # strip the deprecation message - if 'deprecated' in lines[0]: - lines = lines[1:] - assert lines[0] == ( - 'error: attempt to use unversioned python, ' - 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly' - ) - # when the macros are %global, the error is longer - # we deliberately allow this extra line to be optional - if len(lines) > 1 and "error: lua script failed" not in lines[1]: - # the failed macro is not unnecessarily our tested macro - pattern = r'error: Macro %\S+ failed to expand' - assert re.match(pattern, lines[1]) - # but there should be no more lines - assert len(lines) < 3 + assert lines == ['error: attempt to use unversioned python, ' + 'define %__python to /usr/bin/python2 or /usr/bin/python3 explicitly'] @unversioned_macros def test_unversioned_python_works_when_defined(macro): macro3 = macro.replace('python', 'python3').replace('py_', 'py3_') - unverisoned = rpm_eval(macro, __python='/usr/bin/python3') - expected = [l.replace(macro3, macro) for l in rpm_eval(macro3)] - assert unverisoned == expected + assert rpm_eval(macro, __python='/usr/bin/python3') == rpm_eval(macro3) # we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 @@ -950,58 +874,3 @@ def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_s # Compare the last line of the command, that's where lua part is evaluated expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' assert lines[-1].strip() == expected - - -def test_multi_python(alt_x_y): - """ - Ensure memoized %python_version works when switching %__python back - and forth. - """ - versions = ['3', alt_x_y, X_Y, '3'] - evals = [] - for version in versions: - evals.extend((f'%global __python /usr/bin/python{version}', '%python_version')) - lines = rpm_eval(evals) - lines = [l for l in lines if l] # strip empty lines generated by %global - assert lines == [X_Y, alt_x_y, X_Y, X_Y] - - -def test_multi_python3(alt_x_y): - """ - Ensure memoized %python3_version works when switching %__python3 back - and forth. - """ - versions = ['3', alt_x_y, X_Y, '3'] - evals = [] - for version in versions: - evals.extend((f'%global __python3 /usr/bin/python{version}', '%python3_version')) - lines = rpm_eval(evals) - lines = [l for l in lines if l] # strip empty lines generated by %global - assert lines == [X_Y, alt_x_y, X_Y, X_Y] - - -@pytest.mark.parametrize('macro', [ - '%py3_build', - '%py3_build_wheel', - '%py3_install', -]) -def test_deprecation(macro): - lines = rpm_eval(macro) - assert "is deprecated" in lines[0] - assert f"{macro} " in lines[0] - - -def test_multiple_deprecation(): - source = '%{py3_build}' * 10 + '%{py3_build_wheel}' * 10 + '%{py3_install}' * 10 + '%{py3_build}' - lines = rpm_eval(source) - - assert "is deprecated" in lines[0] - assert "%py3_build " in lines[0] - - assert "is deprecated" in lines[1] - assert "%py3_build_wheel " in lines[1] - - assert "is deprecated" in lines[2] - assert "%py3_install " in lines[2] - - assert "is deprecated" not in '\n'.join(lines[3:]) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index dd10689..52e1d7e 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -1,4 +1,4 @@ -from import_all_modules import argparser, exclude_unwanted_module_globs, import_modules +from import_all_modules import argparser, exclude_unwanted_module_globs from import_all_modules import main as modules_main from import_all_modules import read_modules_from_cli, filter_top_level_modules_only @@ -119,7 +119,7 @@ def test_import_all_modules_does_not_import(): # We already imported it in this file once, make sure it's not imported # from the cache sys.modules.pop('import_all_modules') - with pytest.raises(SystemExit): + with pytest.raises(ModuleNotFoundError): modules_main(['import_all_modules']) @@ -127,7 +127,7 @@ def test_modules_from_cwd_not_found(tmp_path, monkeypatch): test_module = tmp_path / 'this_is_a_module_in_cwd.py' test_module.write_text('') monkeypatch.chdir(tmp_path) - with pytest.raises(SystemExit): + with pytest.raises(ModuleNotFoundError): modules_main(['this_is_a_module_in_cwd']) @@ -141,15 +141,15 @@ def test_modules_from_sys_path_found(tmp_path): def test_modules_from_file_are_found(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' - test_file.write_text('math\nwave\ncsv\n') + test_file.write_text('math\nwave\nsunau\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'csv'): + for m in ('math', 'wave', 'sunau'): sys.modules.pop(m, None) modules_main(['-f', str(test_file)]) - assert 'csv' in sys.modules + assert 'sunau' in sys.modules assert 'math' in sys.modules assert 'wave' in sys.modules @@ -160,22 +160,22 @@ def test_modules_from_files_are_found(tmp_path): test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' test_file_1.write_text('math\nwave\n') - test_file_2.write_text('csv\nnetrc\n') - test_file_3.write_text('logging\ncsv\n') + test_file_2.write_text('sunau\npathlib\n') + test_file_3.write_text('logging\nsunau\n') # Make sure the tested modules are not already in sys.modules - for m in ('math', 'wave', 'csv', 'netrc', 'logging'): + for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): sys.modules.pop(m, None) modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) - for module in ('csv', 'math', 'wave', 'netrc', 'logging'): + for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): assert module in sys.modules def test_nonexisting_modules_raise_exception_on_import(tmp_path): test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' test_file.write_text('nonexisting_module\nanother\n') - with pytest.raises(SystemExit): + with pytest.raises(ModuleNotFoundError): modules_main(['-f', str(test_file)]) @@ -203,7 +203,7 @@ def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): sys.path.append(str(tmp_path)) monkeypatch.chdir(cwd_path) - with pytest.raises(SystemExit): + with pytest.raises(ModuleNotFoundError): modules_main([ 'this_is_a_module_in_level_0', 'nested.this_is_a_module_in_level_1', @@ -253,70 +253,24 @@ def test_non_existing_module_raises_exception(tmp_path): test_module_1.write_text('') sys.path.append(str(tmp_path)) - with pytest.raises(SystemExit): + with pytest.raises(ModuleNotFoundError): modules_main([ 'this_is_a_module_in_tmp_path_1', 'this_is_a_module_in_tmp_path_2', ]) -def test_import_module_returns_failed_modules(tmp_path): - test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' - test_module_1.write_text('') - sys.path.append(str(tmp_path)) - - failed_modules = import_modules([ - 'this_is_a_module_in_tmp_path_1', - 'this_is_a_module_in_tmp_path_2', - ]) - - assert failed_modules == ['this_is_a_module_in_tmp_path_2'] - - -def test_module_with_error_propagates_exception(tmp_path, capsys): +def test_module_with_error_propagates_exception(tmp_path): test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' test_module_1.write_text('0/0') sys.path.append(str(tmp_path)) - with pytest.raises(SystemExit): + # The correct exception must be raised + with pytest.raises(ZeroDivisionError): modules_main([ 'this_is_a_module_in_tmp_path_1', ]) - _, err = capsys.readouterr() - assert "ZeroDivisionError" in err - - -def test_import_module_returns_empty_list_when_no_modules_failed(tmp_path): - test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' - test_module_1.write_text('') - sys.path.append(str(tmp_path)) - - failed_modules = import_modules(['this_is_a_module_in_tmp_path_1']) - assert failed_modules == [] - - -def test_all_modules_are_imported(tmp_path, capsys): - test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' - test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' - test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' - - for module in (test_module_1, test_module_2, test_module_3): - module.write_text('') - - sys.path.append(str(tmp_path)) - - with pytest.raises(SystemExit): - modules_main([ - 'this_is_a_module_in_tmp_path_1', - 'missing_module', - 'this_is_a_module_in_tmp_path_2', - 'this_is_a_module_in_tmp_path_3', - ]) - _, err = capsys.readouterr() - for i in range(1, 4): - assert f"Check import: this_is_a_module_in_tmp_path_{i}" in err - assert "Failed to import: missing_module" in err def test_correct_modules_are_excluded(tmp_path): diff --git a/tests/test_rpm_in_distinfo.py b/tests/test_rpm_in_distinfo.py deleted file mode 100644 index 7546dc0..0000000 --- a/tests/test_rpm_in_distinfo.py +++ /dev/null @@ -1,41 +0,0 @@ -from pathlib import Path - -import os -import pytest -import subprocess - -@pytest.fixture -def create_test_files(tmp_path): - def _create(subpath, installer_content): - dir_path = tmp_path / subpath - dir_path.mkdir(parents=True, exist_ok=True) - installer_file = dir_path / "INSTALLER" - installer_file.write_text(installer_content) - record_file = dir_path / "RECORD" - record_file.write_text("dummy content in RECORD file\n") - return dir_path - return _create - -testdata = [ - ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), - ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/", "pip\n", "rpm\n", False), - ("usr/lib/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), - ("usr/lib64/python3.13/site-packages/setuptools/_vendor/zipp-3.19.2.dist-info/", "pip\n", "pip\n", True), - ("usr/lib/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), - ("usr/lib64/python3.13/site-packages/zipp-3.19.2.dist-info/","not pip in INSTALLER\n", "not pip in INSTALLER\n", True), -] -@pytest.mark.parametrize("path, installer_content, expected_installer_content, record_file_exists", testdata) -def test_installer_file_was_correctly_modified(monkeypatch, create_test_files, -path, installer_content, expected_installer_content, record_file_exists): - script_path = Path("/usr/lib/rpm/redhat/brp-python-rpm-in-distinfo") - tmp_dir = create_test_files(path, installer_content) - monkeypatch.setenv("RPM_BUILD_ROOT", str(tmp_dir)) - result = subprocess.run( - [script_path], - capture_output=True, text=True - ) - - assert result.returncode == 0 - assert (Path(tmp_dir) / "INSTALLER").read_text() == expected_installer_content - assert Path(tmp_dir / "RECORD").exists() is record_file_exists - diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..11bb3ae --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,30 @@ +--- +- hosts: localhost + tags: + - classic + tasks: + - dnf: + name: "*" + state: latest + +- hosts: localhost + roles: + - role: standard-test-basic + tags: + - classic + tests: + - pytest: + dir: . + run: PYTHONPATH=/usr/lib/rpm/redhat ALTERNATE_PYTHON_VERSION=3.6 pytest -v + - manual_byte_compilation: + dir: . + run: rpmbuild -ba pythontest.spec + required_packages: + - rpm-build + - python-rpm-macros + - python3-rpm-macros + - python3-devel + - python3-pytest + - python3.6 + - python2.7 + diff --git a/tests/testwheel.spec b/tests/testwheel.spec deleted file mode 100644 index 461d7f3..0000000 --- a/tests/testwheel.spec +++ /dev/null @@ -1,116 +0,0 @@ -Name: testwheel -Epoch: 42 -Version: 1 -Release: 0%{?dist} -Summary: ... -License: MIT -BuildArch: noarch -BuildRequires: python3-devel -BuildRequires: python3-setuptools >= 61 -BuildRequires: python3-pip - -%description -This builds and installs a wheel which we can then use as a test for -%%python_wheel_inject_sbom. - - -%prep -cat > pyproject.toml << EOF -[project] -name = "testwheel" -version = "1" - -[build-system] -requires = ["setuptools >= 61"] -build-backend = "setuptools.build_meta" - -[tool.setuptools] -include-package-data = true - -[tool.setuptools.packages.find] -include = ["testwheel*"] -EOF -# create a secondary dist-info folder in the project -# we need to ensure this file is not altered -mkdir -p testwheel/_vendor/dependency-2.2.2.dist-info -touch testwheel/_vendor/dependency-2.2.2.dist-info/RECORD -echo 'recursive-include testwheel/_vendor *' > MANIFEST.in - - -%build -export PIP_CONFIG_FILE=/dev/null -%{python3} -m pip wheel . --no-build-isolation - -# The macro should happily alter multiple wheels, let's make more -for i in {1..5}; do - mkdir ${i} - cp -a *.whl ${i} -done - -# using relative paths should succeed -%python_wheel_inject_sbom {1..5}/*.whl - -# repetitive use should bail out and fail (SBOM is already there) -%{python_wheel_inject_sbom {1..5}/*.whl} && exit 1 || true - -# each wheel should already have it, all should fail individually as well -for i in {1..5}; do - %{python_wheel_inject_sbom ${i}/*.whl} && exit 1 || true -done - - -%install -mkdir -p %{buildroot}%{python_wheel_dir} -cp -a *.whl %{buildroot}%{python_wheel_dir} - -# using absolute paths should work -%python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl - -# and fail when repeated -%{python_wheel_inject_sbom %{buildroot}%{python_wheel_dir}/*.whl} && exit 1 || true - - -%check -%define venvsite venv/lib/python%{python3_version}/site-packages -%{python3} -m venv venv -venv/bin/pip install --no-index --no-cache-dir %{buildroot}%{python_wheel_dir}/*.whl - -test -f %{venvsite}/testwheel-1.dist-info/RECORD -test -f %{venvsite}/testwheel-1.dist-info/sboms/bom.json -grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD -# a more specific grep. we don't care about CRLF line ends (pip uses those? without the sed the $ doesn't match line end) -sed 's/\r//g' %{venvsite}/testwheel-1.dist-info/RECORD | grep -E '^testwheel-1.dist-info/sboms/bom.json,sha256=[a-f0-9]{64},[0-9]+$' - -test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/RECORD -test -f %{venvsite}/testwheel/_vendor/dependency-2.2.2.dist-info/sboms/bom.json && exit 1 || true - -# this deliberately uses a different mechanism than the macro -# if you are running this test on a different distro, adjust it -%define ns %{?fedora:fedora}%{?eln:fedora}%{?epel:epel}%{!?eln:%{!?epel:%{?rhel:redhat}}} - -PYTHONOPTIMIZE=0 %{python3} -c " -import json -with open('%{venvsite}/testwheel-1.dist-info/sboms/bom.json') as fp: - sbom = json.load(fp) -assert len(sbom['components']) == 1 -assert sbom['components'][0]['type'] == 'library' -assert sbom['components'][0]['name'] == 'testwheel' -assert sbom['components'][0]['version'] == '1-0%{?dist}' -assert sbom['components'][0]['purl'] == 'pkg:rpm/%{ns}/testwheel@1-0%{?dist}?epoch=42&arch=src' -" - -# replace the installation with the original unaltered wheel -venv/bin/pip install --force-reinstall --no-index --no-cache-dir *.whl -test -f %{venvsite}/testwheel-1.dist-info/RECORD -# no SBOM -test ! -e %{venvsite}/testwheel-1.dist-info/sboms/bom.json -grep '^testwheel-1.dist-info/sboms/bom.json,' %{venvsite}/testwheel-1.dist-info/RECORD && exit 1 || true - - -%files -%{python_wheel_dir}/*.whl - - -%changelog -* Wed Aug 13 2025 Miro Hrončok - 42:1-0 -- A static changelog with a date, so we can clamp mtimes