From 68618631a964c815488cf58b2cd7d3b23bbe8689 Mon Sep 17 00:00:00 2001 From: Tomas Orsava Date: Wed, 29 Sep 2021 12:47:19 +0200 Subject: [PATCH 01/12] Define a new macros %python_wheel_dir and %python_wheel_pkg_prefix --- macros.python-srpm | 14 ++++++++++++++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 18 ++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index f115a85..dcc7023 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -49,6 +49,20 @@ # Alternatively, it can be overridden in spec (e.g. to "3.8") when building for alternate Python stacks. %python3_pkgversion 3 +# Define where Python wheels will be stored and the prefix of -wheel packages +# - In Fedora we want wheel subpackages named e.g. `python-pip-wheel` that +# install packages into `/usr/share/python-wheels`. Both names are not +# versioned, because they're used by all Python 3 stacks. +# - In RHEL we want wheel packages named e.g. `python3-pip-wheel` and +# `python3.11-pip-wheel` that install packages into similarly versioned +# locations. We want each Python stack in RHEL to have their own wheels, +# because the main python3 wheels (which we can't upgrade) will likely be +# quite old by the time we're adding new alternate Python stacks. +# - In ELN we want to follow Fedora, because builds for ELN and Fedora rawhide +# need to be interoperable. +%python_wheel_pkg_prefix python%{?rhel:%{!?eln:%{python3_pkgversion}}} +%python_wheel_dir %{_datadir}/%{python_wheel_pkg_prefix}-wheels + ### BRP scripts (and related macros) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 937ea48..22cb056 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -47,7 +47,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 7%{?dist} +Release: 8%{?dist} BuildArch: noarch @@ -141,6 +141,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Wed Sep 29 2021 Tomas Orsava - 3.9-8 +- Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix + * Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild diff --git a/tests/test_evals.py b/tests/test_evals.py index 631f234..887ad30 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -195,6 +195,24 @@ def test_py_provides_with_evr(): assert len(lines) == 3 +def test_python_wheel_pkg_prefix(): + assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None) == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln='1') == ['python'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None) == ['python3'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['python3.10'] + assert rpm_eval('%python_wheel_pkg_prefix', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['python3.11'] + + +def test_python_wheel_dir(): + assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None) == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora='44', rhel=None, eln=None, python3_pkgversion='3.9') == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln='1') == ['/usr/share/python-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None) == ['/usr/share/python3-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.10') == ['/usr/share/python3.10-wheels'] + assert rpm_eval('%python_wheel_dir', fedora=None, rhel='1', eln=None, python3_pkgversion='3.11') == ['/usr/share/python3.11-wheels'] + + def test_pytest_passes_options_naturally(): lines = rpm_eval('%pytest -k foo') assert '/usr/bin/pytest -k foo' in lines[-1] From 9ce6cb80da2c6addeb94288610e9568957076fb2 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 18 Oct 2021 16:33:04 +0200 Subject: [PATCH 02/12] Add new options for %%py{3}_check_import: -f, -t, -e -f: optionally read a file with module names to test -t: bool flag - if set, filter only top-level modules -e: optionally exclude module names matching the given glob (Unix shell-style wildcards) Importing all modules may cause bogus failures in some cases, eg. when the imported code assumes there is an existing graphical window. Such behaviour may be by design, hence for automatic processing it's more convinient to - in some cases - check only for top-level modules or filter out the troublemakers. --- import_all_modules.py | 152 ++++++++++++ macros.python | 13 +- macros.python3 | 13 +- python-rpm-macros.spec | 12 +- tests/test_evals.py | 30 +-- tests/test_import_all_modules.py | 392 +++++++++++++++++++++++++++++++ tests/tests.yml | 2 +- 7 files changed, 586 insertions(+), 28 deletions(-) create mode 100644 import_all_modules.py create mode 100644 tests/test_import_all_modules.py diff --git a/import_all_modules.py b/import_all_modules.py new file mode 100644 index 0000000..7536133 --- /dev/null +++ b/import_all_modules.py @@ -0,0 +1,152 @@ +'''Script to perform import of each module given to %%py_check_import +''' +import argparse +import importlib +import fnmatch +import re +import sys + +from contextlib import contextmanager +from pathlib import Path + + +def read_modules_files(file_paths): + '''Read module names from the files (modules must be newline separated). + + Return the module names list or, if no files were provided, an empty list. + ''' + + if not file_paths: + return [] + + modules = [] + for file in file_paths: + file_contents = file.read_text() + modules.extend(file_contents.split()) + return modules + + +def read_modules_from_cli(argv): + '''Read module names from command-line arguments (space or comma separated). + + Return the module names list. + ''' + + if not argv: + return [] + + # %%py3_check_import allows to separate module list with comma or whitespace, + # we need to unify the output to a list of particular elements + modules_as_str = ' '.join(argv) + modules = re.split(r'[\s,]+', modules_as_str) + return modules + + +def filter_top_level_modules_only(modules): + '''Filter out entries with nested modules (containing dot) ie. 'foo.bar'. + + Return the list of top-level modules. + ''' + + return [module for module in modules if '.' not in module] + + +def any_match(text, globs): + '''Return True if any of given globs fnmatchcase's the given text.''' + + return any(fnmatch.fnmatchcase(text, g) for g in globs) + + +def exclude_unwanted_module_globs(globs, modules): + '''Filter out entries which match the either of the globs given as argv. + + Return the list of filtered modules. + ''' + + return [m for m in modules if not any_match(m, globs)] + + +def read_modules_from_all_args(args): + '''Return a joined list of modules from all given command-line arguments. + ''' + + modules = read_modules_files(args.filename) + modules.extend(read_modules_from_cli(args.modules)) + if args.exclude: + modules = exclude_unwanted_module_globs(args.exclude, modules) + + if args.top_level: + modules = filter_top_level_modules_only(modules) + + # Error when someone accidentally managed to filter out everything + if len(modules) == 0: + raise ValueError('No modules to check were left') + + return modules + + +def import_modules(modules): + '''Procedure to perform import check for each module name from the given list of modules. + ''' + + for module in modules: + print('Check import:', module, file=sys.stderr) + importlib.import_module(module) + + +def argparser(): + parser = argparse.ArgumentParser( + description='Generate list of all importable modules for import check.' + ) + parser.add_argument( + 'modules', nargs='*', + help=('Add modules to check the import (space or comma separated).'), + ) + parser.add_argument( + '-f', '--filename', action='append', type=Path, + help='Add importable module names list from file.', + ) + parser.add_argument( + '-t', '--top-level', action='store_true', + help='Check only top-level modules.', + ) + parser.add_argument( + '-e', '--exclude', action='append', + help='Provide modules globs to be excluded from the check.', + ) + return parser + + +@contextmanager +def remove_unwanteds_from_sys_path(): + '''Remove cwd and this script's parent from sys.path for the import test. + Bring the original contents back after import is done (or failed) + ''' + + cwd_absolute = Path.cwd().absolute() + this_file_parent = Path(__file__).parent.absolute() + old_sys_path = list(sys.path) + for path in old_sys_path: + if Path(path).absolute() in (cwd_absolute, this_file_parent): + sys.path.remove(path) + try: + yield + finally: + sys.path = old_sys_path + + +def main(argv=None): + + cli_args = argparser().parse_args(argv) + + if not cli_args.modules and not cli_args.filename: + raise ValueError('No modules to check were provided') + + modules = read_modules_from_all_args(cli_args) + + with remove_unwanteds_from_sys_path(): + import_modules(modules) + + +if __name__ == '__main__': + main() diff --git a/macros.python b/macros.python index 04c47c5..0f1afa7 100644 --- a/macros.python +++ b/macros.python @@ -67,16 +67,17 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python module(s). +# try to import the Python module(s) given as command-line args or read from file (-f). +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} } %python_provide() %{lua: diff --git a/macros.python3 b/macros.python3 index 93b4199..eac0c00 100644 --- a/macros.python3 +++ b/macros.python3 @@ -65,16 +65,17 @@ } # With $PATH and $PYTHONPATH set to the %%buildroot, -# try to import the given Python 3 module(s). +# try to import the Python 3 module(s) given as command-line args or read from file (-f). +# Filter and check import on only top-level modules using -t flag. +# Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. -# Use spaces or commas as separators. -%py3_check_import() %{expand:\\\ - (cd %{_topdir} &&\\\ +# Use spaces or commas as separators if providing list directly. +# Use newlines as separators if providing list in a file. +%py3_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -c "import %{lua:local m=rpm.expand('%{?*}'):gsub('[%s,]+', ', ');print(m)}" - ) + %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} } # This only supports Python 3.5+ and will never work with Python 2. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 22cb056..92cb7cd 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -15,6 +15,7 @@ Source201: python.lua # Python code %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py +Source302: import_all_modules.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -31,6 +32,7 @@ Source402: brp-python-hardlink Source403: brp-fix-pyc-reproducibility # macros and lua: MIT +# import_all_modules.py: MIT # compileall2.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -47,7 +49,8 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 8%{?dist} +Release: 9%{?dist} + BuildArch: noarch @@ -105,6 +108,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ +install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -131,6 +135,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py +%{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -141,6 +146,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Oct 25 2021 Karolina Surma - 3.10-9 +- Introduce -f (read from file) option to %%py{3}_check_import +- Introduce -t (filter top-level modules) option to %%py{3}_check_import +- Introduce -e (exclude module globs) option to %%py{3}_check_import + * Wed Sep 29 2021 Tomas Orsava - 3.9-8 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix diff --git a/tests/test_evals.py b/tests/test_evals.py index 887ad30..b7746e3 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -623,23 +623,27 @@ def test_python3_sitearch_value(lib): @pytest.mark.parametrize( - 'args, imports', + 'args', [ - ('six', 'six'), - ('five six seven', 'five, six, seven'), - ('six,seven, eight', 'six, seven, eight'), - ('six.quarter six.half,, SIX', 'six.quarter, six.half, SIX'), + 'six', + '-f foo.txt', + '-t -f foo.txt six, seven', + '-e "foo*" -f foo.txt six, seven', + 'six.quarter six.half,, SIX', ] ) -@pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/python3.6']) -def test_py3_check_import(args, imports, __python3, lib): +@pytest.mark.parametrize('__python3', + [None, + f'/usr/bin/python{X_Y}', + '/usr/bin/python3.6']) +def test_py3_check_import(args, __python3, lib): x_y = X_Y - macors = { + macros = { 'buildroot': 'BUILDROOT', - '_topdir': 'TOPDIR', + '_rpmconfigdir': 'RPMCONFIGDIR', } if __python3 is not None: - macors['__python3'] = __python3 + macros['__python3'] = __python3 # If the __python3 command has version at the end, parse it and expect it. # Note that the command is used to determine %python3_sitelib and %python3_sitearch, # so we only test known CPython schemes here and not PyPy for simplicity. @@ -647,7 +651,7 @@ def test_py3_check_import(args, imports, __python3, lib): if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) - lines = rpm_eval(f'%py3_check_import {args}', **macors) + lines = rpm_eval(f'%py3_check_import {args}', **macros) # An equality check is a bit inflexible here, # every time we change the macro we need to change this test. @@ -655,11 +659,9 @@ def test_py3_check_import(args, imports, __python3, lib): # At least, let's make the lines saner to check: lines = [line.rstrip('\\').strip() for line in lines] expected = textwrap.dedent(fr""" - (cd TOPDIR && PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -c "import {imports}" - ) + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {args} """) assert lines == expected.splitlines() diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py new file mode 100644 index 0000000..490ef1d --- /dev/null +++ b/tests/test_import_all_modules.py @@ -0,0 +1,392 @@ +from import_all_modules import argparser, exclude_unwanted_module_globs +from import_all_modules import main as modules_main +from import_all_modules import read_modules_from_cli, filter_top_level_modules_only + +from pathlib import Path + +import pytest +import shlex +import sys + + +@pytest.fixture(autouse=True) +def preserve_sys_path(): + original_sys_path = list(sys.path) + yield + sys.path = original_sys_path + + +@pytest.fixture(autouse=True) +def preserve_sys_modules(): + original_sys_modules = dict(sys.modules) + yield + sys.modules = original_sys_modules + + +@pytest.mark.parametrize( + 'args, imports', + [ + ('six', ['six']), + ('five six seven', ['five', 'six', 'seven']), + ('six,seven, eight', ['six', 'seven', 'eight']), + ('six.quarter six.half,, SIX', ['six.quarter', 'six.half', 'SIX']), + ] +) +def test_read_modules_from_cli(args, imports): + argv = shlex.split(args) + cli_args = argparser().parse_args(argv) + assert read_modules_from_cli(cli_args.modules) == imports + + +@pytest.mark.parametrize( + 'all_mods, imports', + [ + (['six'], ['six']), + (['five', 'six', 'seven'], ['five', 'six', 'seven']), + (['six.seven', 'eight'], ['eight']), + (['SIX', 'six.quarter', 'six.half.and.sth', 'seven'], ['SIX', 'seven']), + ], +) +def test_filter_top_level_modules_only(all_mods, imports): + assert filter_top_level_modules_only(all_mods) == imports + + +@pytest.mark.parametrize( + 'globs, expected', + [ + (['*.*'], ['foo', 'boo']), + (['?oo'], ['foo.bar', 'foo.bar.baz', 'foo.baz']), + (['*.baz'], ['foo', 'foo.bar', 'boo']), + (['foo'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']), + (['foo*'], ['boo']), + (['foo*', '*bar'], ['boo']), + (['foo', 'bar'], ['foo.bar', 'foo.bar.baz', 'foo.baz', 'boo']), + (['*'], []), + ] +) +def test_exclude_unwanted_module_globs(globs, expected): + my_modules = ['foo', 'foo.bar', 'foo.bar.baz', 'foo.baz', 'boo'] + tested = exclude_unwanted_module_globs(globs, my_modules) + assert tested == expected + + +def test_cli_with_all_args(): + '''A smoke test, all args must be parsed correctly.''' + mods = ['foo', 'foo.bar', 'baz'] + files = ['-f', './foo'] + top = ['-t'] + exclude = ['-e', 'foo*'] + cli_args = argparser().parse_args([*mods, *files, *top, *exclude]) + + assert cli_args.filename == [Path('foo')] + assert cli_args.top_level is True + assert cli_args.modules == ['foo', 'foo.bar', 'baz'] + assert cli_args.exclude == ['foo*'] + + +def test_cli_without_filename_toplevel(): + '''Modules provided on command line (without files) must be parsed correctly.''' + mods = ['foo', 'foo.bar', 'baz'] + cli_args = argparser().parse_args(mods) + + assert cli_args.filename is None + assert cli_args.top_level is False + assert cli_args.modules == ['foo', 'foo.bar', 'baz'] + + +def test_cli_with_filename_no_cli_mods(): + '''Files (without any modules provided on command line) must be parsed correctly.''' + + files = ['-f', './foo', '-f', './bar', '-f', './baz'] + cli_args = argparser().parse_args(files) + + assert cli_args.filename == [Path('foo'), Path('./bar'), Path('./baz')] + assert not cli_args.top_level + + +def test_main_raises_error_when_no_modules_provided(): + '''If no filename nor modules were provided, ValueError is raised.''' + + with pytest.raises(ValueError): + modules_main([]) + + +def test_import_all_modules_does_not_import(): + '''Ensure the files from /usr/lib/rpm/redhat cannot be imported and + checked for import''' + + # We already imported it in this file once, make sure it's not imported + # from the cache + sys.modules.pop('import_all_modules') + with pytest.raises(ModuleNotFoundError): + modules_main(['import_all_modules']) + + +def test_modules_from_cwd_not_found(tmp_path, monkeypatch): + test_module = tmp_path / 'this_is_a_module_in_cwd.py' + test_module.write_text('') + monkeypatch.chdir(tmp_path) + with pytest.raises(ModuleNotFoundError): + modules_main(['this_is_a_module_in_cwd']) + + +def test_modules_from_sys_path_found(tmp_path): + test_module = tmp_path / 'this_is_a_module_in_sys_path.py' + test_module.write_text('') + sys.path.append(str(tmp_path)) + modules_main(['this_is_a_module_in_sys_path']) + assert 'this_is_a_module_in_sys_path' in sys.modules + + +def test_modules_from_file_are_found(tmp_path): + test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' + test_file.write_text('math\nwave\nsunau\n') + + # Make sure the tested modules are not already in sys.modules + for m in ('math', 'wave', 'sunau'): + sys.modules.pop(m, None) + + modules_main(['-f', str(test_file)]) + + assert 'sunau' in sys.modules + assert 'math' in sys.modules + assert 'wave' in sys.modules + + +def test_modules_from_files_are_found(tmp_path): + test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' + test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' + test_file_3 = tmp_path / 'this_is_a_file_in_tmp_path_3.txt' + + test_file_1.write_text('math\nwave\n') + test_file_2.write_text('sunau\npathlib\n') + test_file_3.write_text('logging\nsunau\n') + + # Make sure the tested modules are not already in sys.modules + for m in ('math', 'wave', 'sunau', 'pathlib', 'logging'): + sys.modules.pop(m, None) + + modules_main(['-f', str(test_file_1), '-f', str(test_file_2), '-f', str(test_file_3), ]) + for module in ('sunau', 'math', 'wave', 'pathlib', 'logging'): + assert module in sys.modules + + +def test_nonexisting_modules_raise_exception_on_import(tmp_path): + test_file = tmp_path / 'this_is_a_file_in_tmp_path.txt' + test_file.write_text('nonexisting_module\nanother\n') + with pytest.raises(ModuleNotFoundError): + modules_main(['-f', str(test_file)]) + + +def test_nested_modules_found_when_expected(tmp_path, monkeypatch, capsys): + + # This one is supposed to raise an error + cwd_path = tmp_path / 'test_cwd' + Path.mkdir(cwd_path) + test_module_1 = cwd_path / 'this_is_a_module_in_cwd.py' + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_2 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_3 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_4 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3, test_module_4): + module.write_text('') + + sys.path.append(str(tmp_path)) + monkeypatch.chdir(cwd_path) + + with pytest.raises(ModuleNotFoundError): + modules_main([ + 'this_is_a_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + 'this_is_a_module_in_cwd']) + + _, err = capsys.readouterr() + assert 'Check import: this_is_a_module_in_level_0' in err + assert 'Check import: nested.this_is_a_module_in_level_1' in err + assert 'Check import: nested.more_nested.this_is_a_module_in_level_2' in err + assert 'Check import: this_is_a_module_in_cwd' in err + + +def test_modules_both_from_files_and_cli_are_imported(tmp_path): + test_file_1 = tmp_path / 'this_is_a_file_in_tmp_path_1.txt' + test_file_1.write_text('this_is_a_module_in_tmp_path_1') + + test_file_2 = tmp_path / 'this_is_a_file_in_tmp_path_2.txt' + test_file_2.write_text('this_is_a_module_in_tmp_path_2') + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'this_is_a_module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'this_is_a_module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + modules_main([ + '-f', str(test_file_1), + 'this_is_a_module_in_tmp_path_3', + '-f', str(test_file_2), + ]) + + expected = ( + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + 'this_is_a_module_in_tmp_path_3', + ) + for module in expected: + assert module in sys.modules + + +def test_non_existing_module_raises_exception(tmp_path): + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('') + sys.path.append(str(tmp_path)) + + with pytest.raises(ModuleNotFoundError): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + 'this_is_a_module_in_tmp_path_2', + ]) + + +def test_module_with_error_propagates_exception(tmp_path): + + test_module_1 = tmp_path / 'this_is_a_module_in_tmp_path_1.py' + test_module_1.write_text('0/0') + sys.path.append(str(tmp_path)) + + # The correct exception must be raised + with pytest.raises(ZeroDivisionError): + modules_main([ + 'this_is_a_module_in_tmp_path_1', + ]) + + +def test_correct_modules_are_excluded(tmp_path): + test_module_1 = tmp_path / 'module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt' + test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n') + + modules_main([ + '-e', 'module_in_tmp_path_2', + '-f', str(test_file_1), + '-e', 'module_in_tmp_path_3', + ]) + + assert 'module_in_tmp_path_1' in sys.modules + assert 'module_in_tmp_path_2' not in sys.modules + assert 'module_in_tmp_path_3' not in sys.modules + + +def test_excluding_all_modules_raises_error(tmp_path): + test_module_1 = tmp_path / 'module_in_tmp_path_1.py' + test_module_2 = tmp_path / 'module_in_tmp_path_2.py' + test_module_3 = tmp_path / 'module_in_tmp_path_3.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + test_file_1 = tmp_path / 'a_file_in_tmp_path_1.txt' + test_file_1.write_text('module_in_tmp_path_1\nmodule_in_tmp_path_2\nmodule_in_tmp_path_3\n') + + with pytest.raises(ValueError): + modules_main([ + '-e', 'module_in_tmp_path*', + '-f', str(test_file_1), + ]) + + +def test_only_toplevel_modules_found(tmp_path): + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3): + module.write_text('') + + sys.path.append(str(tmp_path)) + + modules_main([ + 'this_is_a_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + '-t']) + + assert 'nested.this_is_a_module_in_level_1' not in sys.modules + assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules + + +def test_only_toplevel_included_modules_found(tmp_path): + + # Nested structure that is supposed to be importable + nested_path_1 = tmp_path / 'nested' + nested_path_2 = nested_path_1 / 'more_nested' + + for path in (nested_path_1, nested_path_2): + Path.mkdir(path) + + test_module_1 = tmp_path / 'this_is_a_module_in_level_0.py' + test_module_4 = tmp_path / 'this_is_another_module_in_level_0.py' + + test_module_2 = nested_path_1 / 'this_is_a_module_in_level_1.py' + test_module_3 = nested_path_2 / 'this_is_a_module_in_level_2.py' + + for module in (test_module_1, test_module_2, test_module_3, test_module_4): + module.write_text('') + + sys.path.append(str(tmp_path)) + + modules_main([ + 'this_is_a_module_in_level_0', + 'this_is_another_module_in_level_0', + 'nested.this_is_a_module_in_level_1', + 'nested.more_nested.this_is_a_module_in_level_2', + '-t', + '-e', '*another*' + ]) + + assert 'nested.this_is_a_module_in_level_1' not in sys.modules + assert 'nested.more_nested.this_is_a_module_in_level_2' not in sys.modules + assert 'this_is_another_module_in_level_0' not in sys.modules + assert 'this_is_a_module_in_level_0' in sys.modules + + +def test_module_list_from_relative_path(tmp_path, monkeypatch): + + monkeypatch.chdir(tmp_path) + test_file_1 = Path('this_is_a_file_in_cwd.txt') + test_file_1.write_text('wave') + + sys.modules.pop('wave', None) + + modules_main([ + '-f', 'this_is_a_file_in_cwd.txt' + ]) + + assert 'wave' in sys.modules diff --git a/tests/tests.yml b/tests/tests.yml index d6090ec..28a9bac 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -15,7 +15,7 @@ tests: - pytest: dir: . - run: pytest -v + run: PYTHONPATH=/usr/lib/rpm/redhat pytest -v - manual_byte_compilation: dir: . run: rpmbuild -ba pythontest.spec From f6c1d3b3d19c1fe5bf8ee5735834eaa5abfd3bbe Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Wed, 27 Oct 2021 08:59:30 +0200 Subject: [PATCH 03/12] Fix changelog entries --- python-rpm-macros.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 92cb7cd..e9f4f34 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -151,10 +151,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ - Introduce -t (filter top-level modules) option to %%py{3}_check_import - Introduce -e (exclude module globs) option to %%py{3}_check_import -* Wed Sep 29 2021 Tomas Orsava - 3.9-8 +* Wed Sep 29 2021 Tomas Orsava - 3.10-8 - Define a new macros %%python_wheel_dir and %%python_wheel_pkg_prefix -* Fri Jul 23 2021 Fedora Release Engineering - 3.9-7 +* Fri Jul 23 2021 Fedora Release Engineering - 3.10-7 - Rebuilt for https://fedoraproject.org/wiki/Fedora_35_Mass_Rebuild * Wed Jul 07 2021 Miro Hrončok - 3.10-6 From a3ce0389932144d2141e88348adf2ec4c2e21e6e Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 1 Nov 2021 09:42:39 +0100 Subject: [PATCH 04/12] Allow multiline arguments processing for %%py3_check_import Fixes the regression introduced with the macro reimplementation. Resolves: rhbz#2018809 --- import_all_modules.py | 4 ++++ macros.python | 7 +++++-- macros.python3 | 7 +++++-- python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 22 +++++++++++++--------- tests/test_import_all_modules.py | 1 + 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/import_all_modules.py b/import_all_modules.py index 7536133..5788b8c 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -39,6 +39,10 @@ def read_modules_from_cli(argv): # we need to unify the output to a list of particular elements modules_as_str = ' '.join(argv) modules = re.split(r'[\s,]+', modules_as_str) + # Because of shell expansion in some less typical cases it may happen + # that a trailing space will occur at the end of the list. + # Remove the empty items from the list before passing it further + modules = [m for m in modules if m] return modules diff --git a/macros.python b/macros.python index 0f1afa7..8ec9e1b 100644 --- a/macros.python +++ b/macros.python @@ -77,8 +77,11 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} -} + %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ + %{lua: + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) + }} %python_provide() %{lua: local python = require "fedora.srpm.python" diff --git a/macros.python3 b/macros.python3 index eac0c00..6546a48 100644 --- a/macros.python3 +++ b/macros.python3 @@ -75,8 +75,11 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py %{?**} -} + %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ + %{lua: + -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) + }} # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e9f4f34..c403f66 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 9%{?dist} +Release: 10%{?dist} BuildArch: noarch @@ -146,6 +146,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Nov 01 2021 Karolina Surma - 3.10-10 +- Fix multiline arguments processing for %%py_check_import +Resolves: rhbz#2018809 + * Mon Oct 25 2021 Karolina Surma - 3.10-9 - Introduce -f (read from file) option to %%py{3}_check_import - Introduce -t (filter top-level modules) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index b7746e3..4542790 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -623,20 +623,22 @@ def test_python3_sitearch_value(lib): @pytest.mark.parametrize( - 'args', + 'args, expected_args', [ - 'six', - '-f foo.txt', - '-t -f foo.txt six, seven', - '-e "foo*" -f foo.txt six, seven', - 'six.quarter six.half,, SIX', + ('six', 'six'), + ('-f foo.txt', '-f foo.txt'), + ('-t -f foo.txt six, seven', '-t -f foo.txt six, seven'), + ('-e "foo*" -f foo.txt six, seven', '-e "foo*" -f foo.txt six, seven'), + ('six.quarter six.half,, SIX', 'six.quarter six.half,, SIX'), + ('-f foo.txt six\nsix.half\nSIX', '-f foo.txt six six.half SIX'), + ('six \\ -e six.half', 'six -e six.half'), ] ) @pytest.mark.parametrize('__python3', [None, f'/usr/bin/python{X_Y}', '/usr/bin/python3.6']) -def test_py3_check_import(args, __python3, lib): +def test_py3_check_import(args, expected_args, __python3, lib): x_y = X_Y macros = { 'buildroot': 'BUILDROOT', @@ -651,7 +653,8 @@ def test_py3_check_import(args, __python3, lib): if (match := re.match(r'.+python(\d+\.\d+)$', __python3)): x_y = match.group(1) - lines = rpm_eval(f'%py3_check_import {args}', **macros) + invocation = '%{py3_check_import ' + args +'}' + lines = rpm_eval(invocation, **macros) # An equality check is a bit inflexible here, # every time we change the macro we need to change this test. @@ -662,6 +665,7 @@ def test_py3_check_import(args, __python3, lib): PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {args} + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py + {expected_args} """) assert lines == expected.splitlines() diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index 490ef1d..d60f6e4 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -30,6 +30,7 @@ def preserve_sys_modules(): ('five six seven', ['five', 'six', 'seven']), ('six,seven, eight', ['six', 'seven', 'eight']), ('six.quarter six.half,, SIX', ['six.quarter', 'six.half', 'SIX']), + ('six.quarter six.half,, SIX \\ ', ['six.quarter', 'six.half', 'SIX']), ] ) def test_read_modules_from_cli(args, imports): From f836bce1c4e048c613d97a8e0496f8a632e25c4b Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Mon, 1 Nov 2021 13:17:42 +0100 Subject: [PATCH 05/12] Fix %%py_shebang_flags handling within %%py_check_import %%py{3}_check_import now respects the custom setting of %%py{3}_shebang_flags and invokes Python with the respective values. If %%py{3}_shebang_flags is undefined or set to no value, there no flags are passed to Python on invoke. Resolves: rhbz#2018615 --- macros.python | 13 ++++++++++--- macros.python3 | 13 ++++++++++--- python-rpm-macros.spec | 2 ++ tests/test_evals.py | 25 +++++++++++++++++++++++-- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/macros.python b/macros.python index 8ec9e1b..c52a170 100644 --- a/macros.python +++ b/macros.python @@ -68,6 +68,7 @@ # With $PATH and $PYTHONPATH set to the %%buildroot, # try to import the Python module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py_shebang_flags or set nothing if it's undefined. # Filter and check import on only top-level modules using -t flag. # Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. @@ -77,11 +78,17 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python} -%{py_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ %{lua: + local command = "%{__python} " + if rpm.expand("%{?py_shebang_flags}") ~= "" then + command = command .. "-%{py_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 - local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) - }} + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } +} %python_provide() %{lua: local python = require "fedora.srpm.python" diff --git a/macros.python3 b/macros.python3 index 6546a48..7956095 100644 --- a/macros.python3 +++ b/macros.python3 @@ -66,6 +66,7 @@ # With $PATH and $PYTHONPATH set to the %%buildroot, # try to import the Python 3 module(s) given as command-line args or read from file (-f). +# Respect the custom values of %%py3_shebang_flags or set nothing if it's undefined. # Filter and check import on only top-level modules using -t flag. # Exclude unwanted modules by passing their globs to -e option. # Useful as a smoke test in %%check when running tests is not feasible. @@ -75,11 +76,17 @@ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ - %{__python3} -%{py3_shebang_flags} %{_rpmconfigdir}/redhat/import_all_modules.py\\\ %{lua: + local command = "%{__python3} " + if rpm.expand("%{?py3_shebang_flags}") ~= "" then + command = command .. "-%{py3_shebang_flags}" + end + command = command .. " %{_rpmconfigdir}/redhat/import_all_modules.py " -- handle multiline arguments correctly, see https://bugzilla.redhat.com/2018809 - local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ");print(args) - }} + local args=rpm.expand('%{?**}'):gsub("[%s\\\\]*%s+", " ") + print(command .. args) + } +} # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c403f66..fea2a50 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -149,6 +149,8 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ * Mon Nov 01 2021 Karolina Surma - 3.10-10 - Fix multiline arguments processing for %%py_check_import Resolves: rhbz#2018809 +- Fix %%py_shebang_flags handling within %%py_check_import +Resolves: rhbz#2018615 * Mon Oct 25 2021 Karolina Surma - 3.10-9 - Introduce -f (read from file) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index 4542790..3971a09 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -643,6 +643,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): macros = { 'buildroot': 'BUILDROOT', '_rpmconfigdir': 'RPMCONFIGDIR', + 'py3_shebang_flags': 's', } if __python3 is not None: macros['__python3'] = __python3 @@ -665,7 +666,27 @@ def test_py3_check_import(args, expected_args, __python3, lib): PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" PYTHONDONTWRITEBYTECODE=1 - {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py - {expected_args} + {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) assert lines == expected.splitlines() + + +@pytest.mark.parametrize( + 'shebang_flags_value, expected_shebang_flags', + [ + ('s', '-s'), + ('%{nil}', ''), + (None, ''), + ('Es', '-Es'), + ] +) +def test_py3_check_import_respects_shebang_flags(shebang_flags_value, expected_shebang_flags, lib): + macros = { + '_rpmconfigdir': 'RPMCONFIGDIR', + '__python3': '/usr/bin/python3', + 'py3_shebang_flags': shebang_flags_value, + } + lines = rpm_eval('%py3_check_import sys', **macros) + # Compare the last line of the command, that's where lua part is evaluated + expected = f'/usr/bin/python3 {expected_shebang_flags} RPMCONFIGDIR/redhat/import_all_modules.py sys' + assert lines[-1].strip() == expected From ff161e10598333878bbf4869ccc297588e44c267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Nov 2021 15:53:43 +0100 Subject: [PATCH 06/12] %py(3)_check_import: Process .pth files in site(arch|lib) Fixes https://bugzilla.redhat.com/show_bug.cgi?id=2018551 --- import_all_modules.py | 15 +++++++++++++++ macros.python | 1 + macros.python3 | 1 + python-rpm-macros.spec | 2 ++ tests/test_evals.py | 1 + tests/test_import_all_modules.py | 33 ++++++++++++++++++++++++++++++++ 6 files changed, 53 insertions(+) diff --git a/import_all_modules.py b/import_all_modules.py index 5788b8c..3930236 100644 --- a/import_all_modules.py +++ b/import_all_modules.py @@ -3,7 +3,9 @@ import argparse import importlib import fnmatch +import os import re +import site import sys from contextlib import contextmanager @@ -139,6 +141,18 @@ def remove_unwanteds_from_sys_path(): sys.path = old_sys_path +def addsitedirs_from_environ(): + '''Load directories from the _PYTHONSITE environment variable (separated by :) + and load the ones already present in sys.path via site.addsitedir() + to handle .pth files in them. + + This is needed to properly import old-style namespace packages with nspkg.pth files. + See https://bugzilla.redhat.com/2018551 for a more detailed rationale.''' + for path in os.getenv('_PYTHONSITE', '').split(':'): + if path in sys.path: + site.addsitedir(path) + + def main(argv=None): cli_args = argparser().parse_args(argv) @@ -149,6 +163,7 @@ def main(argv=None): modules = read_modules_from_all_args(cli_args) with remove_unwanteds_from_sys_path(): + addsitedirs_from_environ() import_modules(modules) diff --git a/macros.python b/macros.python index c52a170..9597506 100644 --- a/macros.python +++ b/macros.python @@ -77,6 +77,7 @@ %py_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python_sitearch}:%{buildroot}%{python_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{lua: local command = "%{__python} " diff --git a/macros.python3 b/macros.python3 index 7956095..f3ff5c0 100644 --- a/macros.python3 +++ b/macros.python3 @@ -75,6 +75,7 @@ %py3_check_import(e:tf:) %{expand:\\\ PATH="%{buildroot}%{_bindir}:$PATH"\\\ PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + _PYTHONSITE="%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}"\\\ PYTHONDONTWRITEBYTECODE=1\\\ %{lua: local command = "%{__python3} " diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index fea2a50..10e287d 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -151,6 +151,8 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ Resolves: rhbz#2018809 - Fix %%py_shebang_flags handling within %%py_check_import Resolves: rhbz#2018615 +- Process .pth files in buildroot's sitedirs in %%py_check_import +Resolves: rhbz#2018551 * Mon Oct 25 2021 Karolina Surma - 3.10-9 - Introduce -f (read from file) option to %%py{3}_check_import diff --git a/tests/test_evals.py b/tests/test_evals.py index 3971a09..732b951 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -665,6 +665,7 @@ def test_py3_check_import(args, expected_args, __python3, lib): expected = textwrap.dedent(fr""" PATH="BUILDROOT/usr/bin:$PATH" PYTHONPATH="${{PYTHONPATH:-BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages}}" + _PYTHONSITE="BUILDROOT/usr/{lib}/python{x_y}/site-packages:BUILDROOT/usr/lib/python{x_y}/site-packages" PYTHONDONTWRITEBYTECODE=1 {__python3 or '/usr/bin/python3'} -s RPMCONFIGDIR/redhat/import_all_modules.py {expected_args} """) diff --git a/tests/test_import_all_modules.py b/tests/test_import_all_modules.py index d60f6e4..52e1d7e 100644 --- a/tests/test_import_all_modules.py +++ b/tests/test_import_all_modules.py @@ -391,3 +391,36 @@ def test_module_list_from_relative_path(tmp_path, monkeypatch): ]) assert 'wave' in sys.modules + + +@pytest.mark.parametrize('arch_in_path', [True, False]) +def test_pth_files_are_read_from__PYTHONSITE(arch_in_path, tmp_path, monkeypatch, capsys): + sitearch = tmp_path / 'lib64' + sitearch.mkdir() + sitelib = tmp_path / 'lib' + sitelib.mkdir() + + for where, word in (sitearch, "ARCH"), (sitelib, "LIB"), (sitelib, "MOD"): + module = where / f'print{word}.py' + module.write_text(f'print("{word}")') + + pth_sitearch = sitearch / 'ARCH.pth' + pth_sitearch.write_text('import printARCH\n') + + pth_sitelib = sitelib / 'LIB.pth' + pth_sitelib.write_text('import printLIB\n') + + if arch_in_path: + sys.path.append(str(sitearch)) + sys.path.append(str(sitelib)) + + # we always add sitearch to _PYTHONSITE + # but when not in sys.path, it should not be processed for .pth files + monkeypatch.setenv('_PYTHONSITE', f'{sitearch}:{sitelib}') + + modules_main(['printMOD']) + out, err = capsys.readouterr() + if arch_in_path: + assert out == 'ARCH\nLIB\nMOD\n' + else: + assert out == 'LIB\nMOD\n' From f0c6736d3bfa75811e8ae894992309b3b7dc3fc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 1 Nov 2021 16:04:22 +0100 Subject: [PATCH 07/12] Move import_all_modules out of python-srpm-macros There's no need for it in the default buildroot. --- python-rpm-macros.spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 10e287d..a2dedad 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -87,7 +87,7 @@ Summary: RPM macros for building Python 3 packages # For %%__python3 and %%python3 Requires: python-srpm-macros = %{version}-%{release} -# For %%py_setup +# For %%py_setup and import_all_modules.py Requires: python-rpm-macros = %{version}-%{release} %description -n python3-rpm-macros @@ -131,11 +131,11 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %files %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile +%{_rpmconfigdir}/redhat/import_all_modules.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm %{_rpmconfigdir}/redhat/compileall2.py -%{_rpmconfigdir}/redhat/import_all_modules.py %{_rpmconfigdir}/redhat/brp-python-bytecompile %{_rpmconfigdir}/redhat/brp-python-hardlink %{_rpmconfigdir}/redhat/brp-fix-pyc-reproducibility @@ -153,6 +153,7 @@ Resolves: rhbz#2018809 Resolves: rhbz#2018615 - Process .pth files in buildroot's sitedirs in %%py_check_import Resolves: rhbz#2018551 +- Move import_all_modules.py from python-srpm-macros to python-rpm-macros * Mon Oct 25 2021 Karolina Surma - 3.10-9 - Introduce -f (read from file) option to %%py{3}_check_import From cb6359bbddf803a48894687ba586c7cc7d1a52d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 3 Sep 2021 16:06:03 +0200 Subject: [PATCH 08/12] Set $RPM_BUILD_ROOT in %{python3_...} macros, for alternate sysconfig install scheme Our Pythons currently patches distutils to install packages to /usr/lib(64)/pythonX.Y/site-packages when the $RPM_BUILD_ROOT environment variable is set (and to /usr/local/lib(64)/pythonX.Y/site-packages otherwise). With the deprecation of distutils [1] we want to change the patch to create and use a different sysconfig install scheme [2]. However, we have realized that macros defined as %(%{__python3} ...) don't "see" the environment variables set by rpmbuild because they are expanded earlier and hence e.g. %{python3_sitelib} evaluates to /usr/local/lib/python3.X/site-packages -- which is not desired. To be able to reliably detect an RPM build environment by checking the presence of the $RPM_BUILD_ROOT environment variable, we manually set it in the macro definitions. Since %{buildroot} in not fully populated (e.g. it can expand literally to /home/anna/rpmbuild/BUILDROOT/%{NAME}-%{VERSION}-%{RELEASE}.x86_64), we don't use it here. The variable simply needs to present in the environment. See also the analysis of the build failures when this is not done [3]. [1] https://www.python.org/dev/peps/pep-0632/ [2] https://bugs.python.org/issue43976 [3] https://src.fedoraproject.org/rpms/python3.10/pull-request/63#comment-79042 ------------------------------------------------------------------------------ Note about the backport: This is not required with the default %__python3 value, but it is required when %__python3 is redefined to /usr/bin/python3.11 (or newer). --- macros.python | 16 +++++++++------- macros.python3 | 16 +++++++++------- python-rpm-macros.spec | 6 +++++- 3 files changed, 23 insertions(+), 15 deletions(-) diff --git a/macros.python b/macros.python index 9597506..f0d431d 100644 --- a/macros.python +++ b/macros.python @@ -1,12 +1,14 @@ # unversioned macros: used with user defined __python, no longer part of rpm >= 4.15 # __python is defined to error by default in the srpm macros -%python_sitelib %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") -%python_sitearch %(%{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") -%python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") -%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python_sitelib %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('purelib'))") +%python_sitearch %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_path('platlib'))") +%python_version %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python_version_nodots %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index f3ff5c0..0ccafd9 100644 --- a/macros.python3 +++ b/macros.python3 @@ -1,10 +1,12 @@ -%python3_sitelib %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") -%python3_sitearch %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") -%python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") -%python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") -%python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") -%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") -%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +# nb: $RPM_BUILD_ROOT is not set when the macros are expanded (at spec parse time) +# so we set it manually (to empty string), making our Python prefer the correct install scheme location +%python3_sitelib %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('purelib'))") +%python3_sitearch %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_path('platlib'))") +%python3_version %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") +%python3_version_nodots %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index a2dedad..865caef 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 10%{?dist} +Release: 11%{?dist} BuildArch: noarch @@ -146,6 +146,10 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Mon Feb 07 2022 Miro Hrončok - 3.10-11 +- Set $RPM_BUILD_ROOT in %%{python3_...} macros + to allow selecting alternate sysconfig install scheme based on that variable + * Mon Nov 01 2021 Karolina Surma - 3.10-10 - Fix multiline arguments processing for %%py_check_import Resolves: rhbz#2018809 From bb20c6908e666d138b83c52582a1b176a104df99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 19 Jul 2022 11:14:59 +0000 Subject: [PATCH 09/12] Define %python3_cache_tag / %python_cache_tag, e.g. cpython-311 When reviewing https://src.fedoraproject.org/rpms/pyproject-rpm-macros/pull-request/291 we have discovered that there is no macronized way to get this part of some paths and that packagers need to hardcode it as cpython-%{python3_version_nodots}. This way, we have a standardized macro packagers (and other macros) can use. --- macros.python | 1 + macros.python3 | 2 ++ python-rpm-macros.spec | 5 ++++- tests/test_evals.py | 13 +++++++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index f0d431d..9cec71c 100644 --- a/macros.python +++ b/macros.python @@ -9,6 +9,7 @@ %python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") %python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") %python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index 0ccafd9..a29c5ef 100644 --- a/macros.python3 +++ b/macros.python3 @@ -7,6 +7,7 @@ %python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") %python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") %python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") +%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s @@ -102,6 +103,7 @@ pyminor = path:match("/python3.(%d+)/") or "*" dirname = path:match("(.*/)") modulename = path:match(".*/([^/]+).py") + -- %%python3_cache_tag is not used here because this macro supports not-installed CPythons print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") end } diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 865caef..e3f5b65 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -49,7 +49,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 11%{?dist} +Release: 12%{?dist} BuildArch: noarch @@ -146,6 +146,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Jul 19 2022 Miro Hrončok - 3.10-12 +- Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 + * Mon Feb 07 2022 Miro Hrončok - 3.10-11 - Set $RPM_BUILD_ROOT in %%{python3_...} macros to allow selecting alternate sysconfig install scheme based on that variable diff --git a/tests/test_evals.py b/tests/test_evals.py index 732b951..5060705 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -560,6 +560,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [ '%python_platform', '%python_platform_triplet', '%python_ext_suffix', + '%python_cache_tag', '%py_shebang_fix', '%py_build', '%py_build_egg', @@ -598,6 +599,18 @@ def test_ext_suffix(): assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"] +def test_cache_tag(): + assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"] + + +def test_cache_tag_alternate_python(): + assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python3.6") == [f"cpython-36"] + + +def test_cache_tag_alternate_python3(): + assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python3.6") == [f"cpython-36"] + + def test_python_sitelib_value(): macro = '%python_sitelib' assert rpm_eval(macro, __python='/usr/bin/python3.6') == [f'/usr/lib/python3.6/site-packages'] From d83a32823cf8758cb52bda89d545a8a35dbe6792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 19 Jul 2022 16:34:01 +0200 Subject: [PATCH 10/12] Define and use %{_py3_shebang_s} in the shebang macros for easier opt-out This is in-line with https://fedoraproject.org/wiki/Changes/PythonSafePath#Opting_out Backported from https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/141 --- macros.python | 3 ++- macros.python3 | 3 ++- python-rpm-macros.spec | 1 + tests/test_evals.py | 7 +++++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/macros.python b/macros.python index 9cec71c..71a0bae 100644 --- a/macros.python +++ b/macros.python @@ -12,7 +12,8 @@ %python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)") %py_setup setup.py -%py_shbang_opts -s +%_py_shebang_s s +%py_shbang_opts -%{?_py_shebang_s} %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ diff --git a/macros.python3 b/macros.python3 index a29c5ef..b3a0d2c 100644 --- a/macros.python3 +++ b/macros.python3 @@ -10,7 +10,8 @@ %python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} -%py3_shbang_opts -s +%_py3_shebang_s s +%py3_shbang_opts -%{?_py3_shebang_s} %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e3f5b65..c93d2b2 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -148,6 +148,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog * Tue Jul 19 2022 Miro Hrončok - 3.10-12 - Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 +- Define and use %%{_py3_shebang_s} in the shebang macros for easier opt-out * Mon Feb 07 2022 Miro Hrončok - 3.10-11 - Set $RPM_BUILD_ROOT in %%{python3_...} macros diff --git a/tests/test_evals.py b/tests/test_evals.py index 5060705..43ac6f2 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -334,6 +334,13 @@ def test_py3_shebang_fix_custom_shebang_flags(): assert shell_stdout('\n'.join(lines)) == '-kaEs' +@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}']) +def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s): + lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s) + lines[-1] = 'echo $shebang_flags' + assert shell_stdout('\n'.join(lines)) == '-k' + + @pytest.mark.parametrize('flags', [None, '%{nil}']) def test_py3_shebang_fix_no_shebang_flags(flags): lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags) From b7bcd2829e8f9ffc8680514df839ecbdcb3b2ede Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Fri, 28 Jan 2022 17:04:43 +0100 Subject: [PATCH 11/12] Disable certain rpminspect inspections not relevant to this package --- rpminspect.yaml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 rpminspect.yaml diff --git a/rpminspect.yaml b/rpminspect.yaml new file mode 100644 index 0000000..4589e70 --- /dev/null +++ b/rpminspect.yaml @@ -0,0 +1,7 @@ +# completely disabled inspections: +inspections: + # there is no upstream and the files are changed from time to time + addedfiles: off + changedfiles: off + filesize: off + upstream: off From 14edaf699e2ff509a00e33fa5d6bf092e80690ba Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 25 Oct 2022 10:57:44 +0200 Subject: [PATCH 12/12] Include pathfix.py in this package --- macros.python | 8 +- macros.python3 | 8 +- pathfix.py | 199 +++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 15 +++- tests/test_evals.py | 4 +- 5 files changed, 216 insertions(+), 18 deletions(-) create mode 100644 pathfix.py diff --git a/macros.python b/macros.python index 71a0bae..1d13263 100644 --- a/macros.python +++ b/macros.python @@ -17,18 +17,12 @@ %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) %py_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python_version}.py ]; then - pathfix=/usr/bin/pathfix%{python_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py_shebang_flags}" fi - $pathfix -pni %{__python} $shebang_flags} + %{__python} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index b3a0d2c..264cdca 100644 --- a/macros.python3 +++ b/macros.python3 @@ -15,18 +15,12 @@ %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) %py3_shebang_fix %{expand:\\\ - if [ -f /usr/bin/pathfix%{python3_version}.py ]; then - pathfix=/usr/bin/pathfix%{python3_version}.py - else - # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly - pathfix=/usr/bin/pathfix.py - fi if [ -z "%{?py3_shebang_flags}" ]; then shebang_flags="-k" else shebang_flags="-ka%{py3_shebang_flags}" fi - $pathfix -pni %{__python3} $shebang_flags} + %{__python3} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python3} $shebang_flags} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/pathfix.py b/pathfix.py new file mode 100644 index 0000000..1d7db3a --- /dev/null +++ b/pathfix.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 + +import sys +import os +from stat import * +import getopt + +err = sys.stderr.write +dbg = err +rep = sys.stdout.write + +new_interpreter = None +preserve_timestamps = False +create_backup = True +keep_flags = False +add_flags = b'' + + +def main(): + global new_interpreter + global preserve_timestamps + global create_backup + global keep_flags + global add_flags + + usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' % + sys.argv[0]) + try: + opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn') + except getopt.error as msg: + err(str(msg) + '\n') + err(usage) + sys.exit(2) + for o, a in opts: + if o == '-i': + new_interpreter = a.encode() + if o == '-p': + preserve_timestamps = True + if o == '-n': + create_backup = False + if o == '-k': + keep_flags = True + if o == '-a': + add_flags = a.encode() + if b' ' in add_flags: + err("-a option doesn't support whitespaces") + sys.exit(2) + if not new_interpreter or not new_interpreter.startswith(b'/') or \ + not args: + err('-i option or file-or-directory missing\n') + err(usage) + sys.exit(2) + bad = 0 + for arg in args: + if os.path.isdir(arg): + if recursedown(arg): bad = 1 + elif os.path.islink(arg): + err(arg + ': will not process symbolic links\n') + bad = 1 + else: + if fix(arg): bad = 1 + sys.exit(bad) + + +def ispython(name): + return name.endswith('.py') + + +def recursedown(dirname): + dbg('recursedown(%r)\n' % (dirname,)) + bad = 0 + try: + names = os.listdir(dirname) + except OSError as msg: + err('%s: cannot list directory: %r\n' % (dirname, msg)) + return 1 + names.sort() + subdirs = [] + for name in names: + if name in (os.curdir, os.pardir): continue + fullname = os.path.join(dirname, name) + if os.path.islink(fullname): pass + elif os.path.isdir(fullname): + subdirs.append(fullname) + elif ispython(name): + if fix(fullname): bad = 1 + for fullname in subdirs: + if recursedown(fullname): bad = 1 + return bad + + +def fix(filename): +## dbg('fix(%r)\n' % (filename,)) + try: + f = open(filename, 'rb') + except IOError as msg: + err('%s: cannot open: %r\n' % (filename, msg)) + return 1 + with f: + line = f.readline() + fixed = fixline(line) + if line == fixed: + rep(filename+': no change\n') + return + head, tail = os.path.split(filename) + tempname = os.path.join(head, '@' + tail) + try: + g = open(tempname, 'wb') + except IOError as msg: + err('%s: cannot create: %r\n' % (tempname, msg)) + return 1 + with g: + rep(filename + ': updating\n') + g.write(fixed) + BUFSIZE = 8*1024 + while 1: + buf = f.read(BUFSIZE) + if not buf: break + g.write(buf) + + # Finishing touch -- move files + + mtime = None + atime = None + # First copy the file's mode to the temp file + try: + statbuf = os.stat(filename) + mtime = statbuf.st_mtime + atime = statbuf.st_atime + os.chmod(tempname, statbuf[ST_MODE] & 0o7777) + except OSError as msg: + err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) + # Then make a backup of the original file as filename~ + if create_backup: + try: + os.rename(filename, filename + '~') + except OSError as msg: + err('%s: warning: backup failed (%r)\n' % (filename, msg)) + else: + try: + os.remove(filename) + except OSError as msg: + err('%s: warning: removing failed (%r)\n' % (filename, msg)) + # Now move the temp file to the original file + try: + os.rename(tempname, filename) + except OSError as msg: + err('%s: rename failed (%r)\n' % (filename, msg)) + return 1 + if preserve_timestamps: + if atime and mtime: + try: + os.utime(filename, (atime, mtime)) + except OSError as msg: + err('%s: reset of timestamp failed (%r)\n' % (filename, msg)) + return 1 + # Return success + return 0 + + +def parse_shebang(shebangline): + shebangline = shebangline.rstrip(b'\n') + start = shebangline.find(b' -') + if start == -1: + return b'' + return shebangline[start:] + + +def populate_flags(shebangline): + old_flags = b'' + if keep_flags: + old_flags = parse_shebang(shebangline) + if old_flags: + old_flags = old_flags[2:] + if not (old_flags or add_flags): + return b'' + # On Linux, the entire string following the interpreter name + # is passed as a single argument to the interpreter. + # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s" + # so shebang should have single '-' where flags are given and + # flag might need argument for that reasons adding new flags is + # between '-' and original flags + # e.g. #! /usr/bin/python3 -sW Error + return b' -' + add_flags + old_flags + + +def fixline(line): + if not line.startswith(b'#!'): + return line + + if b"python" not in line: + return line + + flags = populate_flags(line) + return b'#! ' + new_interpreter + flags + b'\n' + + +if __name__ == '__main__': + main() diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index c93d2b2..40bd489 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -16,6 +16,8 @@ Source201: python.lua %global compileall2_version 0.7.1 Source301: https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py Source302: import_all_modules.py +%global pathfix_version 1.0.0 +Source303: https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py # BRP scripts # This one is from redhat-rpm-config < 190 @@ -34,6 +36,7 @@ Source403: brp-fix-pyc-reproducibility # macros and lua: MIT # import_all_modules.py: MIT # compileall2.py: PSFv2 +# pathfix.py: PSFv2 # brp scripts: GPLv2+ License: MIT and Python and GPLv2+ @@ -49,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then end } Version: %{__default_python3_version} -Release: 12%{?dist} +Release: 13%{?dist} BuildArch: noarch @@ -98,6 +101,10 @@ RPM macros for building Python 3 packages. %autosetup -c -T cp -a %{sources} . +# We want to have shebang in the script upstream but not here so +# the package with macros does not depend on Python. +sed -i '1s=^#!/usr/bin/env python3==' pathfix.py + %install mkdir -p %{buildroot}%{rpmmacrodir} @@ -109,7 +116,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua mkdir -p %{buildroot}%{_rpmconfigdir}/redhat install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/ - +install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ @@ -132,6 +139,7 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %{rpmmacrodir}/macros.python %{rpmmacrodir}/macros.pybytecompile %{_rpmconfigdir}/redhat/import_all_modules.py +%{_rpmconfigdir}/redhat/pathfix.py %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm @@ -146,6 +154,9 @@ install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/ %changelog +* Tue Oct 25 2022 Lumír Balhar - 3.10-13 +- Include pathfix.py in this package + * Tue Jul 19 2022 Miro Hrončok - 3.10-12 - Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311 - Define and use %%{_py3_shebang_s} in the shebang macros for easier opt-out diff --git a/tests/test_evals.py b/tests/test_evals.py index 43ac6f2..999b21f 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -319,7 +319,7 @@ def test_pypi_source_explicit_tilde(): def test_py3_shebang_fix(): cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/python3 -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3' def test_py3_shebang_fix_default_shebang_flags(): @@ -350,7 +350,7 @@ def test_py3_shebang_fix_no_shebang_flags(flags): def test_py_shebang_fix_custom_python(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip() - assert cmd == '$pathfix -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' + assert cmd == '/usr/bin/pypy -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3' def test_pycached_in_sitelib():