From f43227fac0ebe73f2b665670ae7e8bcbe4d9745a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 6 Feb 2020 10:24:03 +0100 Subject: [PATCH 01/10] Define %py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a --- macros.python | 1 + macros.python2 | 1 + macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index e431b92..72bb64b 100644 --- a/macros.python +++ b/macros.python @@ -7,6 +7,7 @@ %py_setup setup.py %py_shbang_opts -s +%py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 4ff5f41..4442496 100644 --- a/macros.python2 +++ b/macros.python2 @@ -4,6 +4,7 @@ %python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %py2_shbang_opts -s +%py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index b772fb4..9d61547 100644 --- a/macros.python3 +++ b/macros.python3 @@ -6,6 +6,7 @@ %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s +%py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 79ed93b..e42fb8f 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 51%{?dist} +Release: 52%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Fri Feb 07 2020 Miro Hrončok - 3-52 +- Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a + * Sat Dec 28 2019 Miro Hrončok - 3-51 - Define %%python, but make it work only if %%__python is redefined - Add the %%pycached macro From 5914a75181e34428be9ccb3cb43104419990b284 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 10 Feb 2020 23:47:52 +0100 Subject: [PATCH 02/10] Update of bundled compileall2 module to 0.7.0 Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication This is explained in https://discuss.python.org/t/3014 and https://github.com/fedora-python/compileall2/issues/16 This option is not yet used anywhere. That allows us to backport this to all Fedoras but only use --hardlink-dupes on rawhide first. --- compileall2.py | 46 ++++++++++++++++++++++++++++++++++-------- python-rpm-macros.spec | 8 ++++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/compileall2.py b/compileall2.py index 593dda9..8976fa0 100644 --- a/compileall2.py +++ b/compileall2.py @@ -19,6 +19,7 @@ import sys import importlib.util import py_compile import struct +import filecmp from functools import partial from pathlib import Path @@ -86,7 +87,7 @@ def _walk_dir(dir, maxlevels, quiet=0): def compile_dir(dir, maxlevels=None, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, stripdir=None, - prependdir=None, limit_sl_dest=None): + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -109,6 +110,7 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path + hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None if workers is not None: @@ -144,14 +146,15 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, if not compile_file(file, ddir, force, rx, quiet, legacy, optimize, invalidation_mode, stripdir=stripdir, prependdir=prependdir, - limit_sl_dest=limit_sl_dest): + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): success = False return success def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, stripdir=None, prependdir=None, - limit_sl_dest=None): + limit_sl_dest=None, hardlink_dupes=False): """Byte-compile one file. Arguments (only fullname is required): @@ -172,6 +175,7 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, after stripdir limit_sl_dest: ignore symlinks if they are pointing outside of the defined path. + hardlink_dupes: hardlink duplicated pyc files """ if ddir is not None and (stripdir is not None or prependdir is not None): @@ -212,6 +216,10 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if isinstance(optimize, int): optimize = [optimize] + if hardlink_dupes: + raise ValueError(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if rx is not None: mo = rx.search(fullname) if mo: @@ -256,7 +264,8 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, if not quiet: print('Compiling {!r}...'.format(fullname)) try: - for opt_level, cfile in opt_cfiles.items(): + for index, opt_level in enumerate(sorted(optimize)): + cfile = opt_cfiles[opt_level] if PY37: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level, @@ -264,6 +273,18 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, else: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=opt_level) + + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if previous_cfile == cfile and optimize[0] not in (1, 2): + # Python 3.4 has only one .pyo file for -O and -OO so + # we hardlink it only if there is a .pyc file + # with the same content + previous_cfile = opt_cfiles[optimize[0]] + if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) + except py_compile.PyCompileError as err: success = False if quiet >= 2: @@ -379,11 +400,14 @@ def main(): parser.add_argument('-j', '--workers', default=1, type=int, help='Run compileall concurrently') parser.add_argument('-o', action='append', type=int, dest='opt_levels', - help=('Optimization levels to run compilation with.' - 'Default is -1 which uses optimization level of' + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses optimization level of ' 'Python interpreter itself (specified by -O).')) parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') if PY37: invalidation_modes = [mode.name.lower().replace('_', '-') @@ -413,6 +437,10 @@ def main(): if args.opt_levels is None: args.opt_levels = [-1] + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + if args.ddir is not None and ( args.stripdir is not None or args.prependdir is not None ): @@ -449,7 +477,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False else: if not compile_dir(dest, maxlevels, args.ddir, @@ -459,7 +488,8 @@ def main(): stripdir=args.stripdir, prependdir=args.prependdir, optimize=args.opt_levels, - limit_sl_dest=args.limit_sl_dest): + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): success = False return success else: diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index e42fb8f..9af171c 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 52%{?dist} +Release: 53%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.5.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Feb 10 2020 Miro Hrončok - 3-53 +- Update of bundled compileall2 module to 0.7.0 + Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication + * Fri Feb 07 2020 Miro Hrončok - 3-52 - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a From 3f331a6ad14d53edf31e4f25f8090931a1f02ecd Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 3 Mar 2020 15:26:45 +0100 Subject: [PATCH 03/10] Update of bundled compileall2 module to 0.7.1 (bugfix release) --- compileall2.py | 7 +++++++ python-rpm-macros.spec | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compileall2.py b/compileall2.py index 8976fa0..c58e545 100644 --- a/compileall2.py +++ b/compileall2.py @@ -113,6 +113,13 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + if ddir is not None: + stripdir = dir + prependdir = ddir + ddir = None if workers is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 9af171c..19bc664 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 53%{?dist} +Release: 54%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide and compileall2.py @@ -78,6 +78,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Tue Mar 31 2020 Lumír Balhar - 3-54 +- Update of bundled compileall2 module to 0.7.1 (bugfix release) + * Mon Feb 10 2020 Miro Hrončok - 3-53 - Update of bundled compileall2 module to 0.7.0 Adds the optional --hardlink-dupes flag for compileall2 for pyc deduplication From 18667277d5e0d0ecf90876b06b7109a254046ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 27 Apr 2020 10:40:14 +0200 Subject: [PATCH 04/10] Make pythonX-rpm-macros depend on python-rpm-macros %pyX_(build|install) uses %py_setup from python-rpm-macros Fixes https://bugzilla.redhat.com/show_bug.cgi?id=1827811 --- python-rpm-macros.spec | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 19bc664..8a4637b 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 54%{?dist} +Release: 55%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -35,6 +35,7 @@ RPM macros for building Python source packages. %package -n python2-rpm-macros Summary: RPM macros for building Python 2 packages Requires: python-srpm-macros >= 3-38 +Requires: python-rpm-macros # Would need to be different for each release - worth it? #Conflicts: python2-devel < 2.7.11-3 @@ -44,6 +45,7 @@ RPM macros for building Python 2 packages. %package -n python3-rpm-macros Summary: RPM macros for building Python 3 packages Requires: python-srpm-macros >= 3-38 +Requires: python-rpm-macros %description -n python3-rpm-macros RPM macros for building Python 3 packages. @@ -78,6 +80,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Tue Apr 28 2020 Miro Hrončok - 3-55 +- Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) + * Tue Mar 31 2020 Lumír Balhar - 3-54 - Update of bundled compileall2 module to 0.7.1 (bugfix release) From ead8ec025b7f6f534ff1d2c07e64bf3b7fe876be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 20 May 2020 13:13:22 +0200 Subject: [PATCH 05/10] Backport simplified %py_provides This is a smallest possible backport of https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/52 Notable simplifications: - there is no Lua library - %python_provide remains untouched and the logic is not shared with %py_provides - there are no pythonXY-foo provides on Fedora < 33 --- macros.python-srpm | 17 +++++++++ python-rpm-macros.spec | 5 ++- tests/.gitignore | 1 + tests/test_evals.py | 83 ++++++++++++++++++++++++++++++++++++++++++ tests/tests.yml | 22 +++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/.gitignore create mode 100644 tests/test_evals.py create mode 100644 tests/tests.yml diff --git a/macros.python-srpm b/macros.python-srpm index 9fa4f06..a1b75d6 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -132,3 +132,20 @@ \ print(url .. first .. '/' .. src .. '/' .. src .. '-' .. ver .. '.' .. ext) } + +%py_provides() %{lua: + local name = rpm.expand('%1') + if name == '%1' then + rpm.expand('%{error:%%py_provides requires at least 1 argument, the name to provide}') + end + local evr = rpm.expand('%2') + if evr == '%2' then + evr = rpm.expand('%{?epoch:%{epoch}:}%{version}-%{release}') + end + print('Provides: ' .. name .. ' = ' .. evr .. '\\n') + -- NB: dash needs to be escaped! + if name:match('^python3%-') then + replaced = name:gsub('^python3%-', 'python-') + print('Provides: ' .. replaced .. ' = ' .. evr .. '\\n') + end +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 8a4637b..239c390 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 55%{?dist} +Release: 56%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -80,6 +80,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Wed May 20 2020 Miro Hrončok - 3-56 +- Implement %%py_provides + * Tue Apr 28 2020 Miro Hrončok - 3-55 - Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..5732c0f --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +__*__/ diff --git a/tests/test_evals.py b/tests/test_evals.py new file mode 100644 index 0000000..8547c73 --- /dev/null +++ b/tests/test_evals.py @@ -0,0 +1,83 @@ +import subprocess +import sys + + +def rpm_eval(expression, **kwargs): + cmd = ['rpmbuild'] + for var, value in kwargs.items(): + cmd += ['--define', f'{var} {value}'] + cmd += ['--eval', expression] + cp = subprocess.run(cmd, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert cp.returncode == 0, cp.stderr + return cp.stdout.strip().splitlines() + + +def test_python_provide_python(): + assert rpm_eval('%python_provide python-foo') == [] + + +def test_python_provide_python3(): + lines = rpm_eval('%python_provide python3-foo', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 2 + + +def test_python_provide_python3_epoched(): + lines = rpm_eval('%python_provide python3-foo', epoch='1', version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert len(lines) == 2 + + +def test_python_provide_doubleuse(): + lines = rpm_eval('%{python_provide python3-foo}%{python_provide python3-foo}', + version='6', release='1.fc66') + assert 'Obsoletes: python-foo < 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 4 + assert len(set(lines)) == 2 + + +def test_py_provides_python(): + lines = rpm_eval('%py_provides python-foo', version='6', release='1.fc66') + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 1 + + +def test_py_provides_whatever(): + lines = rpm_eval('%py_provides whatever', version='6', release='1.fc66') + assert 'Provides: whatever = 6-1.fc66' in lines + assert len(lines) == 1 + + +def test_py_provides_python3(): + lines = rpm_eval('%py_provides python3-foo', version='6', release='1.fc66') + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 2 + + +def test_py_provides_python3_epoched(): + lines = rpm_eval('%py_provides python3-foo', epoch='1', version='6', release='1.fc66') + assert 'Provides: python3-foo = 1:6-1.fc66' in lines + assert 'Provides: python-foo = 1:6-1.fc66' in lines + assert len(lines) == 2 + + +def test_py_provides_doubleuse(): + lines = rpm_eval('%{py_provides python3-foo}%{py_provides python3-foo}', + version='6', release='1.fc66') + assert 'Provides: python3-foo = 6-1.fc66' in lines + assert 'Provides: python-foo = 6-1.fc66' in lines + assert len(lines) == 4 + assert len(set(lines)) == 2 + + +def test_py_provides_with_evr(): + lines = rpm_eval('%py_provides python3-foo 123', + version='6', release='1.fc66') + assert 'Provides: python3-foo = 123' in lines + assert 'Provides: python-foo = 123' in lines + assert len(lines) == 2 diff --git a/tests/tests.yml b/tests/tests.yml new file mode 100644 index 0000000..56221e3 --- /dev/null +++ b/tests/tests.yml @@ -0,0 +1,22 @@ +--- +- hosts: localhost + tags: + - classic + tasks: + - dnf: + name: "*" + state: latest + +- hosts: localhost + roles: + - role: standard-test-basic + tags: + - classic + tests: + - pytest: + dir: . + run: pytest -v + required_packages: + - rpm-build + - python-rpm-macros + - python3-pytest From 70fa6a962770fcc05099ba751c65288d7b293db4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 6 May 2020 13:21:32 +0200 Subject: [PATCH 06/10] Implement %pytest See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/XLPDSH362PJKMJCAYOXNJNV53Y66EF6B/ --- macros.python3 | 9 +++++++++ python-rpm-macros.spec | 1 + tests/test_evals.py | 10 ++++++++++ 3 files changed, 20 insertions(+) diff --git a/macros.python3 b/macros.python3 index 9d61547..7c3b26c 100644 --- a/macros.python3 +++ b/macros.python3 @@ -53,3 +53,12 @@ print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") end } + +# This is intended for Python 3 only, hence also no Python version in the name. +%__pytest /usr/bin/pytest +%pytest %{expand:\\\ + CFLAGS="${CFLAGS:-${RPM_OPT_FLAGS}}" LDFLAGS="${LDFLAGS:-${RPM_LD_FLAGS}}"\\\ + PATH="%{buildroot}%{_bindir}:$PATH"\\\ + PYTHONPATH="${PYTHONPATH:-%{buildroot}%{python3_sitearch}:%{buildroot}%{python3_sitelib}}"\\\ + PYTHONDONTWRITEBYTECODE=1\\\ + %__pytest} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 239c390..cc09edd 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -82,6 +82,7 @@ install -m 644 %{SOURCE5} \ %changelog * Wed May 20 2020 Miro Hrončok - 3-56 - Implement %%py_provides +- Implement %%pytest * Tue Apr 28 2020 Miro Hrončok - 3-55 - Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) diff --git a/tests/test_evals.py b/tests/test_evals.py index 8547c73..9e80d29 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -81,3 +81,13 @@ def test_py_provides_with_evr(): assert 'Provides: python3-foo = 123' in lines assert 'Provides: python-foo = 123' in lines assert len(lines) == 2 + + +def test_pytest_passes_options_naturally(): + lines = rpm_eval('%pytest -k foo') + assert '/usr/bin/pytest -k foo' in lines[-1] + + +def test_pytest_different_command(): + lines = rpm_eval('%pytest', __pytest='pytest-3') + assert 'pytest-3' in lines[-1] From c3cafa3dd7b4d644224d0cd5a5c694b843183b71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 11 May 2020 19:02:37 +0200 Subject: [PATCH 07/10] Strip tildes from %version in %pypi_source by default, add tests --- macros.python-srpm | 4 ++-- python-rpm-macros.spec | 1 + tests/test_evals.py | 54 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/macros.python-srpm b/macros.python-srpm index a1b75d6..87aeeb0 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -93,7 +93,7 @@ # Accepts zero to three arguments: # 1: The PyPI project name, defaulting to %srcname if it is defined, then # %pypi_name if it is defined, then just %name. -# 2: The PYPI version, defaulting to %version. +# 2: The PYPI version, defaulting to %version with tildes stripped. # 3: The file extension, defaulting to "tar.gz". (A period will be added # automatically.) # Requires %__pypi_url and %__pypi_default_extension to be defined. @@ -120,7 +120,7 @@ \ -- If no second argument, use %version if ver == '%2' then - ver = rpm.expand('%version') + ver = rpm.expand('%version'):gsub('~', '') end \ -- If no third argument, use the preset default extension diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index cc09edd..0bf9231 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -83,6 +83,7 @@ install -m 644 %{SOURCE5} \ * Wed May 20 2020 Miro Hrončok - 3-56 - Implement %%py_provides - Implement %%pytest +- Strip tildes from %%version in %%pypi_source by default * Tue Apr 28 2020 Miro Hrončok - 3-55 - Make pythonX-rpm-macros depend on python-rpm-macros (#1827811) diff --git a/tests/test_evals.py b/tests/test_evals.py index 9e80d29..a07c676 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -91,3 +91,57 @@ def test_pytest_passes_options_naturally(): def test_pytest_different_command(): lines = rpm_eval('%pytest', __pytest='pytest-3') assert 'pytest-3' in lines[-1] + + +def test_pypi_source_default_name(): + url = rpm_eval('%pypi_source', + name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_srcname(): + url = rpm_eval('%pypi_source', + name='python-foo', srcname='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_pypi_name(): + url = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_default_name_uppercase(): + url = rpm_eval('%pypi_source', + name='Foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/F/Foo/Foo-6.tar.gz' + + +def test_pypi_source_provided_name(): + url = rpm_eval('%pypi_source foo', + name='python-bar', pypi_name='bar', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_provided_name_version(): + url = rpm_eval('%pypi_source foo 6', + name='python-bar', pypi_name='bar', version='3')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.tar.gz' + + +def test_pypi_source_provided_name_version_ext(): + url = rpm_eval('%pypi_source foo 6 zip', + name='python-bar', pypi_name='bar', version='3')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6.zip' + + +def test_pypi_source_prerelease(): + url = rpm_eval('%pypi_source', + name='python-foo', pypi_name='foo', version='6~b2')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6b2.tar.gz' + + +def test_pypi_source_explicit_tilde(): + url = rpm_eval('%pypi_source foo 6~6', + name='python-foo', pypi_name='foo', version='6')[0] + assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz' From 8b79a441adcb77434f0d4c0d49bb559e313df5e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Mon, 18 May 2020 18:58:39 +0200 Subject: [PATCH 08/10] Implement %pyX_shebang_fix See https://lists.fedoraproject.org/archives/list/python-devel@lists.fedoraproject.org/thread/UGCMDDG3S32U7JJK36OEZNHLUVQQRG3M/ --- macros.python | 2 ++ macros.python2 | 2 ++ macros.python3 | 2 ++ python-rpm-macros.spec | 1 + tests/test_evals.py | 25 ++++++++++++++++++++++++- tests/tests.yml | 1 + 6 files changed, 32 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 72bb64b..20cde47 100644 --- a/macros.python +++ b/macros.python @@ -8,6 +8,8 @@ %py_setup setup.py %py_shbang_opts -s %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) +%py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-}) +%py_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python} -k%{?py_shebang_flags:a %py_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 4442496..3426929 100644 --- a/macros.python2 +++ b/macros.python2 @@ -5,6 +5,8 @@ %py2_shbang_opts -s %py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) +%py2_shebang_flags %(opts=%{py2_shbang_opts}; echo ${opts#-}) +%py2_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python2} -k%{?py2_shebang_flags:a %py2_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 7c3b26c..efccee7 100644 --- a/macros.python3 +++ b/macros.python3 @@ -7,6 +7,8 @@ %py3_shbang_opts -s %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) +%py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-}) +%py3_shebang_fix %{expand:/usr/bin/pathfix.py -pni %{__python3} -k%{?py3_shebang_flags:a %py3_shebang_flags}} # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 0bf9231..4c8abd4 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -83,6 +83,7 @@ install -m 644 %{SOURCE5} \ * Wed May 20 2020 Miro Hrončok - 3-56 - Implement %%py_provides - Implement %%pytest +- Implement %%pyX_shebang_fix - Strip tildes from %%version in %%pypi_source by default * Tue Apr 28 2020 Miro Hrončok - 3-55 diff --git a/tests/test_evals.py b/tests/test_evals.py index a07c676..b63e29e 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -5,7 +5,10 @@ import sys def rpm_eval(expression, **kwargs): cmd = ['rpmbuild'] for var, value in kwargs.items(): - cmd += ['--define', f'{var} {value}'] + if value is None: + cmd += ['--undefine', var] + else: + cmd += ['--define', f'{var} {value}'] cmd += ['--eval', expression] cp = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -145,3 +148,23 @@ def test_pypi_source_explicit_tilde(): url = rpm_eval('%pypi_source foo 6~6', name='python-foo', pypi_name='foo', version='6')[0] assert url == 'https://files.pythonhosted.org/packages/source/f/foo/foo-6~6.tar.gz' + + +def test_py3_shebang_fix(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka s arg1 arg2 arg3' + + +def test_py3_shebang_fix_custom_flags(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags='Es')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -ka Es arg1 arg2 arg3' + + +def test_py3_shebang_fix_empty_flags(): + cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3', py3_shebang_flags=None)[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/python3 -k arg1 arg2 arg3' + + +def test_py_shebang_fix_custom(): + cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[0] + assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy -ka s arg1 arg2 arg3' diff --git a/tests/tests.yml b/tests/tests.yml index 56221e3..2411fa8 100644 --- a/tests/tests.yml +++ b/tests/tests.yml @@ -19,4 +19,5 @@ required_packages: - rpm-build - python-rpm-macros + - python3-rpm-macros - python3-pytest From 6186b94b6eb15dddf7ba4d8df5cb3df5e476dcbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Wed, 27 May 2020 10:30:27 +0200 Subject: [PATCH 09/10] Allow to combine %pycached with other macros (e.g. %exclude or %ghost) Previous implementation allowed for only one argument to be passed to the %pycached macro, which made it impossible to combine it with other macros. Current implementation allows to pass other macros as arguments to %pycached. Example: %pycached %exclude /path/to/foo.py For macro expansion limitations, the opposite order is not possible. That is to be documented in the guidelines: https://pagure.io/packaging-committee/pull-request/986 Added some tests. Resolves https://bugzilla.redhat.com/show_bug.cgi?id=1838992 Co-authored-by: Marcel Plch --- macros.python3 | 2 +- python-rpm-macros.spec | 5 +++- tests/test_evals.py | 61 +++++++++++++++++++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 6 deletions(-) diff --git a/macros.python3 b/macros.python3 index efccee7..ea38385 100644 --- a/macros.python3 +++ b/macros.python3 @@ -44,7 +44,7 @@ # This only supports Python 3.5+ and will never work with Python 2. # Hence, it has no Python version in the name. %pycached() %{lua: - path = rpm.expand("%{?1}") + path = rpm.expand("%{?*}") if (string.sub(path, "-3") ~= ".py") then rpm.expand("%{error:%%pycached can only be used with paths explicitly ending with .py}") else diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 4c8abd4..caeef87 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 56%{?dist} +Release: 57%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -80,6 +80,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Mon Jun 15 2020 Miro Hrončok - 3-57 +- Allow to combine %%pycached with other macros (e.g. %%exclude or %%ghost) (#1838992) + * Wed May 20 2020 Miro Hrončok - 3-56 - Implement %%py_provides - Implement %%pytest diff --git a/tests/test_evals.py b/tests/test_evals.py index b63e29e..b585754 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,8 +1,12 @@ +import os import subprocess import sys +X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' +XY = f'{sys.version_info[0]}{sys.version_info[1]}' -def rpm_eval(expression, **kwargs): + +def rpm_eval(expression, fails=False, **kwargs): cmd = ['rpmbuild'] for var, value in kwargs.items(): if value is None: @@ -10,9 +14,12 @@ def rpm_eval(expression, **kwargs): else: cmd += ['--define', f'{var} {value}'] cmd += ['--eval', expression] - cp = subprocess.run(cmd, text=True, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - assert cp.returncode == 0, cp.stderr + cp = subprocess.run(cmd, text=True, env={**os.environ, 'LANG': 'C.utf-8'}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if fails: + assert cp.returncode != 0, cp.stdout + elif fails is not None: + assert cp.returncode == 0, cp.stdout return cp.stdout.strip().splitlines() @@ -168,3 +175,49 @@ def test_py3_shebang_fix_empty_flags(): def test_py_shebang_fix_custom(): cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[0] assert cmd == '/usr/bin/pathfix.py -pni /usr/bin/pypy -ka s arg1 arg2 arg3' + + +def test_pycached_in_sitelib(): + lines = rpm_eval('%pycached %{python3_sitelib}/foo*.py') + assert lines == [ + f'/usr/lib/python{X_Y}/site-packages/foo*.py', + f'/usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_in_sitearch(): + lines = rpm_eval('%pycached %{python3_sitearch}/foo*.py') + lib = rpm_eval('%_lib')[0] + assert lines == [ + f'/usr/{lib}/python{X_Y}/site-packages/foo*.py', + f'/usr/{lib}/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_in_36(): + lines = rpm_eval('%pycached /usr/lib/python3.6/site-packages/foo*.py') + assert lines == [ + '/usr/lib/python3.6/site-packages/foo*.py', + '/usr/lib/python3.6/site-packages/__pycache__/foo*.cpython-36{,.opt-?}.pyc' + ] + + +def test_pycached_in_custom_dir(): + lines = rpm_eval('%pycached /bar/foo*.py') + assert lines == [ + '/bar/foo*.py', + '/bar/__pycache__/foo*.cpython-3*{,.opt-?}.pyc' + ] + + +def test_pycached_with_exclude(): + lines = rpm_eval('%pycached %exclude %{python3_sitelib}/foo*.py') + assert lines == [ + f'%exclude /usr/lib/python{X_Y}/site-packages/foo*.py', + f'%exclude /usr/lib/python{X_Y}/site-packages/__pycache__/foo*.cpython-{XY}{{,.opt-?}}.pyc' + ] + + +def test_pycached_fails_with_extension_glob(): + lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True) + assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py' From a1861889d386d9e3b6952eaa584c22298cbc0686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 24 Sep 2020 21:09:21 +0000 Subject: [PATCH 10/10] Add %python3_platform_triplet and %python3_ext_suffix Also add %python_platform_triplet, %python_ext_suffix and %python_platform. The CI tests are limited to x86_64 for now. https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names --- macros.python | 3 +++ macros.python3 | 2 ++ python-rpm-macros.spec | 6 +++++- tests/test_evals.py | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 20cde47..9e2c37f 100644 --- a/macros.python +++ b/macros.python @@ -4,6 +4,9 @@ %python_sitearch %(%{__python} -Esc "from distutils.sysconfig import get_python_lib; print(get_python_lib(1))") %python_version %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python_version_nodots %(%{__python} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") +%python_platform %(%{__python} -Esc "import sysconfig; print(sysconfig.get_platform())") +%python_platform_triplet %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python_ext_suffix %(%{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py_setup setup.py %py_shbang_opts -s diff --git a/macros.python3 b/macros.python3 index ea38385..9172c13 100644 --- a/macros.python3 +++ b/macros.python3 @@ -3,6 +3,8 @@ %python3_version %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}.{0.minor}'.format(sys.version_info))") %python3_version_nodots %(%{__python3} -Ic "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %python3_platform %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())") +%python3_platform_triplet %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))") +%python3_ext_suffix %(%{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index caeef87..07eb1ed 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 57%{?dist} +Release: 59%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -80,6 +80,10 @@ install -m 644 %{SOURCE5} \ %changelog +* Thu Sep 24 2020 Miro Hrončok - 3-59 +- Add %%python3_platform_triplet and %%python3_ext_suffix +- https://fedoraproject.org/wiki/Changes/Python_Upstream_Architecture_Names + * Mon Jun 15 2020 Miro Hrončok - 3-57 - Allow to combine %%pycached with other macros (e.g. %%exclude or %%ghost) (#1838992) diff --git a/tests/test_evals.py b/tests/test_evals.py index b585754..a65cd9d 100644 --- a/tests/test_evals.py +++ b/tests/test_evals.py @@ -1,7 +1,10 @@ import os import subprocess +import platform import sys +import pytest + X_Y = f'{sys.version_info[0]}.{sys.version_info[1]}' XY = f'{sys.version_info[0]}{sys.version_info[1]}' @@ -221,3 +224,17 @@ def test_pycached_with_exclude(): def test_pycached_fails_with_extension_glob(): lines = rpm_eval('%pycached %{python3_sitelib}/foo.py*', fails=True) assert lines[0] == 'error: %pycached can only be used with paths explicitly ending with .py' + + +# we could rework the test for multiple architectures, but the Fedora CI currently only runs on x86_64 +x86_64_only = pytest.mark.skipif(platform.machine() != "x86_64", reason="works on x86_64 only") + + +@x86_64_only +def test_platform_triplet(): + assert rpm_eval("%python3_platform_triplet")[0] == "x86_64-linux-gnu" + + +@x86_64_only +def test_ext_suffix(): + assert rpm_eval("%python3_ext_suffix")[0] == f".cpython-{XY}m-x86_64-linux-gnu.so"