From 936fd1dd8cadc9c37b1c97d209208956c00461c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 1 Aug 2019 13:48:51 +0200 Subject: [PATCH 1/6] Define %python2 and %python3 See https://pagure.io/packaging-committee/issue/907 Redefine %__pythonX to change the behavior of %pythonX, %pythonX_version, etc. Use %pythonX in spec. --- macros.python-srpm | 6 ++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 514a449..62e7431 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -2,9 +2,15 @@ # - they can be used in Build/Requires # - they can be used in non-Python packages where requiring pythonX-devel would # be an overkill + +# use the underscored macros to redefine the behavior of %%python3_version etc. %__python2 /usr/bin/python2 %__python3 /usr/bin/python3 +# use the non-underscored macros to refer to Python in spec, etc. +%python2 %__python2 +%python3 %__python3 + # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index b2b04bd..04423c5 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 42%{?dist} +Release: 43%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Fri Sep 27 2019 Miro Hrončok - 3-43 +- Define %%python2 and %%python3 + * Sat Feb 02 2019 Fedora Release Engineering - 3-42 - Rebuilt for https://fedoraproject.org/wiki/Fedora_30_Mass_Rebuild From 4559c0e65ffde455bc462f15a27aed5a9c015da4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 28 Dec 2019 19:10:09 +0100 Subject: [PATCH 2/6] Define %python, but make it work only if %__python is redefined --- macros.python-srpm | 10 ++++++++++ python-rpm-macros.spec | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/macros.python-srpm b/macros.python-srpm index 62e7431..4fc56c8 100644 --- a/macros.python-srpm +++ b/macros.python-srpm @@ -11,6 +11,16 @@ %python2 %__python2 %python3 %__python3 +# Users can use %%python only if they redefined %%__python (e.g. to %%__python3) +%python() %{lua:\ + __python = rpm.expand("%__python")\ + if __python == "/usr/bin/python" then\ + rpm.expand("%{error:Cannot use %%python if %%__python wasn't redefined to something other than /usr/bin/python.}")\ + else\ + print(__python)\ + end\ +} + # python3_pkgversion specifies the version of Python 3 in the distro. It can be # a specific version (e.g. 34 in Fedora EPEL7) %python3_pkgversion 3 diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 04423c5..2059ff0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 43%{?dist} +Release: 44%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Sat Dec 28 2019 Miro Hrončok - 3-44 +- Define %%python, but make it work only if %%__python is redefined + * Fri Sep 27 2019 Miro Hrončok - 3-43 - Define %%python2 and %%python3 From 597d3648746a8ae26c10de91b4cb9e7591a9c11e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Sat, 28 Dec 2019 19:12:37 +0100 Subject: [PATCH 3/6] Add the %pycached macro Usage: %files ... %pycached %{python3_sitelib}/foo.py This will list: /usr/lib/python3.8/site-packages/foo.py /usr/lib/python3.8/site-packages/__pycache__/foo.cpython-38{,.opt-?}.pyc Assuming the Python 3 version is 3.8. The bytecode files are globbed, their presence is not checked. This will fail: %pycached %{python3_sitelib}/foo error: %pycached can only be used with paths explicitly ending with .py And so will any of this: %pycached %{python3_sitelib}/* %pycached %{python3_sitelib}/foo.* %pycached %{python3_sitelib}/foo.p? %pycached %{python3_sitelib}/foo.?y %pycached %{python3_sitelib}/foo.?? But this will work: %pycached %{python3_sitelib}/foo*.py And it will generate the following globs: /usr/lib/python3.8/site-packages/foo*.py /usr/lib/python3.8/site-packages/__pycache__/foo*.cpython-38{,.opt-?}.pyc When used with paths that include Python 3 version, it globs with the version: %pycached /opt/python3.10/foo.py Generates: /opt/python3.10/foo.py /opt/python3.10/__pycache__/foo.cpython-310{,.opt-?}.pyc While paths without version have less strict globs: %pycached /custom/foo.py /custom/foo.py /custom/__pycache__/foo.cpython-3*{,.opt-?}.pyc This will generate a warning in RPM build: warning: File listed twice: /custom/__pycache__/foo.cpython-38.opt-1.pyc However it ensures the optimized bytecode is there. --- macros.python3 | 15 +++++++++++++++ python-rpm-macros.spec | 1 + 2 files changed, 16 insertions(+) diff --git a/macros.python3 b/macros.python3 index 1952aed..646ae86 100644 --- a/macros.python3 +++ b/macros.python3 @@ -37,3 +37,18 @@ %py3_install_wheel() %{expand:\\\ pip%{python3_version} install -I dist/%{1} --root %{buildroot} --strip-file-prefix %{buildroot} --no-deps } + +# This only supports Python 3.5+ and will never work with Python 2. +# Hence, it has no Python version in the name. +%pycached() %{lua: + path = rpm.expand("%{?1}") + if (string.sub(path, "-3") ~= ".py") then + rpm.expand("%{error:%%pycached can only be used with paths explicitly ending with .py}") + else + print(path) + pyminor = path:match("/python3.(%d+)/") or "*" + dirname = path:match("(.*/)") + modulename = path:match(".*/([^/]+).py") + print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc") + end +} diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 2059ff0..6134b28 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -75,6 +75,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog * Sat Dec 28 2019 Miro Hrončok - 3-44 - Define %%python, but make it work only if %%__python is redefined +- Add the %%pycached macro * Fri Sep 27 2019 Miro Hrončok - 3-43 - Define %%python2 and %%python3 From 5c8a587f3f6d45c909f8a28861a9df1cd1770baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Thu, 6 Feb 2020 10:24:03 +0100 Subject: [PATCH 4/6] Define %py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a --- macros.python | 1 + macros.python2 | 1 + macros.python3 | 1 + python-rpm-macros.spec | 5 ++++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/macros.python b/macros.python index 8ad1161..1fcea41 100644 --- a/macros.python +++ b/macros.python @@ -1,5 +1,6 @@ %py_setup setup.py %py_shbang_opts -s +%py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python2 b/macros.python2 index 38c6eb9..289bac5 100644 --- a/macros.python2 +++ b/macros.python2 @@ -4,6 +4,7 @@ %python2_version_nodots %(%{__python2} -Esc "import sys; sys.stdout.write('{0.major}{0.minor}'.format(sys.version_info))") %py2_shbang_opts -s +%py2_shbang_opts_nodash %(opts=%{py2_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/macros.python3 b/macros.python3 index 646ae86..97a5089 100644 --- a/macros.python3 +++ b/macros.python3 @@ -6,6 +6,7 @@ %py3dir %{_builddir}/python3-%{name}-%{version}-%{release} %py3_shbang_opts -s +%py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-}) # Use the slashes after expand so that the command starts on the same line as # the macro diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 6134b28..fc089c0 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 44%{?dist} +Release: 45%{?dist} Summary: The unversioned Python RPM macros License: MIT @@ -73,6 +73,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Fri Feb 07 2020 Miro Hrončok - 3-45 +- Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a + * Sat Dec 28 2019 Miro Hrončok - 3-44 - Define %%python, but make it work only if %%__python is redefined - Add the %%pycached macro From 28ed6130e0209f6f7f3741ce23a23d01afe63e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Tue, 11 Feb 2020 00:04:52 +0100 Subject: [PATCH 5/6] Add the compileall2 module (0.7.0) to be used in various Python spec files No macros here use it, but since we updated python38, python39 to use it, we need the module if we don't want to diverge the branches of python3Y needlessly. --- compileall2.py | 508 +++++++++++++++++++++++++++++++++++++++++ python-rpm-macros.spec | 18 +- 2 files changed, 522 insertions(+), 4 deletions(-) create mode 100644 compileall2.py diff --git a/compileall2.py b/compileall2.py new file mode 100644 index 0000000..8976fa0 --- /dev/null +++ b/compileall2.py @@ -0,0 +1,508 @@ +"""Module/script to byte-compile all .py files to .pyc files. + +When called as a script with arguments, this compiles the directories +given as arguments recursively; the -l option prevents it from +recursing into directories. + +Without arguments, if compiles all modules on sys.path, without +recursing into subdirectories. (Even though it should do so for +packages -- for now, you'll have to deal with packages separately.) + +See module py_compile for details of the actual byte-compilation. + +License: +Compileall2 is an enhanced copy of Python's compileall module +and it follows Python licensing. For more info see: https://www.python.org/psf/license/ +""" +import os +import sys +import importlib.util +import py_compile +import struct +import filecmp + +from functools import partial +from pathlib import Path + +# Python 3.7 and higher +PY37 = sys.version_info[0:2] >= (3, 7) +# Python 3.6 and higher +PY36 = sys.version_info[0:2] >= (3, 6) +# Python 3.5 and higher +PY35 = sys.version_info[0:2] >= (3, 5) + +# Python 3.7 and above has a different structure and length +# of pyc files header. Also, multiple ways how to invalidate pyc file was +# introduced in Python 3.7. These cases are covered by variables here or by PY37 +# variable itself. +if PY37: + pyc_struct_format = '<4sll' + pyc_header_lenght = 12 + pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER, 0) +else: + pyc_struct_format = '<4sl' + pyc_header_lenght = 8 + pyc_header_format = (pyc_struct_format, importlib.util.MAGIC_NUMBER) + +__all__ = ["compile_dir","compile_file","compile_path"] + +def optimization_kwarg(opt): + """Returns opt as a dictionary {optimization: opt} for use as **kwarg + for Python >= 3.5 and empty dictionary for Python 3.4""" + if PY35: + return dict(optimization=opt) + else: + # `debug_override` is a way how to enable optimized byte-compiled files + # (.pyo) in Python <= 3.4 + if opt: + return dict(debug_override=False) + else: + return dict() + +def _walk_dir(dir, maxlevels, quiet=0): + if PY36 and quiet < 2 and isinstance(dir, os.PathLike): + dir = os.fspath(dir) + else: + dir = str(dir) + if not quiet: + print('Listing {!r}...'.format(dir)) + try: + names = os.listdir(dir) + except OSError: + if quiet < 2: + print("Can't list {!r}".format(dir)) + names = [] + names.sort() + for name in names: + if name == '__pycache__': + continue + fullname = os.path.join(dir, name) + if not os.path.isdir(fullname): + yield fullname + elif (maxlevels > 0 and name != os.curdir and name != os.pardir and + os.path.isdir(fullname) and not os.path.islink(fullname)): + yield from _walk_dir(fullname, maxlevels=maxlevels - 1, + quiet=quiet) + +def compile_dir(dir, maxlevels=None, ddir=None, force=False, + rx=None, quiet=0, legacy=False, optimize=-1, workers=1, + invalidation_mode=None, stripdir=None, + prependdir=None, limit_sl_dest=None, hardlink_dupes=False): + """Byte-compile all modules in the given directory tree. + + Arguments (only dir is required): + + dir: the directory to byte-compile + maxlevels: maximum recursion level (default `sys.getrecursionlimit()`) + ddir: the directory that will be prepended to the path to the + file as it is compiled into each byte-code file. + force: if True, force compilation, even if timestamps are up-to-date + quiet: full output with False or 0, errors only with 1, + no output with 2 + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. + workers: maximum number of parallel workers + invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path + hardlink_dupes: hardlink duplicated pyc files + """ + ProcessPoolExecutor = None + if workers is not None: + if workers < 0: + raise ValueError('workers must be greater or equal to 0') + elif workers != 1: + try: + # Only import when needed, as low resource platforms may + # fail to import it + from concurrent.futures import ProcessPoolExecutor + except ImportError: + workers = 1 + if maxlevels is None: + maxlevels = sys.getrecursionlimit() + files = _walk_dir(dir, quiet=quiet, maxlevels=maxlevels) + success = True + if workers is not None and workers != 1 and ProcessPoolExecutor is not None: + workers = workers or None + with ProcessPoolExecutor(max_workers=workers) as executor: + results = executor.map(partial(compile_file, + ddir=ddir, force=force, + rx=rx, quiet=quiet, + legacy=legacy, + optimize=optimize, + invalidation_mode=invalidation_mode, + stripdir=stripdir, + prependdir=prependdir, + limit_sl_dest=limit_sl_dest), + files) + success = min(results, default=True) + else: + for file in files: + if not compile_file(file, ddir, force, rx, quiet, + legacy, optimize, invalidation_mode, + stripdir=stripdir, prependdir=prependdir, + limit_sl_dest=limit_sl_dest, + hardlink_dupes=hardlink_dupes): + success = False + return success + +def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, + legacy=False, optimize=-1, + invalidation_mode=None, stripdir=None, prependdir=None, + limit_sl_dest=None, hardlink_dupes=False): + """Byte-compile one file. + + Arguments (only fullname is required): + + fullname: the file to byte-compile + ddir: if given, the directory name compiled in to the + byte-code file. + force: if True, force compilation, even if timestamps are up-to-date + quiet: full output with False or 0, errors only with 1, + no output with 2 + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths + optimize: int or list of optimization levels or -1 for level of + the interpreter. Multiple levels leads to multiple compiled + files each with one optimization level. + invalidation_mode: how the up-to-dateness of the pyc will be checked + stripdir: part of path to left-strip from source file path + prependdir: path to prepend to beggining of original file path, applied + after stripdir + limit_sl_dest: ignore symlinks if they are pointing outside of + the defined path. + hardlink_dupes: hardlink duplicated pyc files + """ + + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + + success = True + if PY36 and quiet < 2 and isinstance(fullname, os.PathLike): + fullname = os.fspath(fullname) + else: + fullname = str(fullname) + name = os.path.basename(fullname) + + dfile = None + + if ddir is not None: + if not PY36: + ddir = str(ddir) + dfile = os.path.join(ddir, name) + + if stripdir is not None: + fullname_parts = fullname.split(os.path.sep) + stripdir_parts = stripdir.split(os.path.sep) + ddir_parts = list(fullname_parts) + + for spart, opart in zip(stripdir_parts, fullname_parts): + if spart == opart: + ddir_parts.remove(spart) + + dfile = os.path.join(*ddir_parts) + + if prependdir is not None: + if dfile is None: + dfile = os.path.join(prependdir, fullname) + else: + dfile = os.path.join(prependdir, dfile) + + if isinstance(optimize, int): + optimize = [optimize] + + if hardlink_dupes: + raise ValueError(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + + if rx is not None: + mo = rx.search(fullname) + if mo: + return success + + if limit_sl_dest is not None and os.path.islink(fullname): + if Path(limit_sl_dest).resolve() not in Path(fullname).resolve().parents: + return success + + opt_cfiles = {} + + if os.path.isfile(fullname): + for opt_level in optimize: + if legacy: + opt_cfiles[opt_level] = fullname + 'c' + else: + if opt_level >= 0: + opt = opt_level if opt_level >= 1 else '' + opt_kwarg = optimization_kwarg(opt) + cfile = (importlib.util.cache_from_source( + fullname, **opt_kwarg)) + opt_cfiles[opt_level] = cfile + else: + cfile = importlib.util.cache_from_source(fullname) + opt_cfiles[opt_level] = cfile + + head, tail = name[:-3], name[-3:] + if tail == '.py': + if not force: + try: + mtime = int(os.stat(fullname).st_mtime) + expect = struct.pack(*(pyc_header_format + (mtime,))) + for cfile in opt_cfiles.values(): + with open(cfile, 'rb') as chandle: + actual = chandle.read(pyc_header_lenght) + if expect != actual: + break + else: + return success + except OSError: + pass + if not quiet: + print('Compiling {!r}...'.format(fullname)) + try: + for index, opt_level in enumerate(sorted(optimize)): + cfile = opt_cfiles[opt_level] + if PY37: + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=opt_level, + invalidation_mode=invalidation_mode) + else: + ok = py_compile.compile(fullname, cfile, dfile, True, + optimize=opt_level) + + if index > 0 and hardlink_dupes: + previous_cfile = opt_cfiles[optimize[index - 1]] + if previous_cfile == cfile and optimize[0] not in (1, 2): + # Python 3.4 has only one .pyo file for -O and -OO so + # we hardlink it only if there is a .pyc file + # with the same content + previous_cfile = opt_cfiles[optimize[0]] + if previous_cfile != cfile and filecmp.cmp(cfile, previous_cfile, shallow=False): + os.unlink(cfile) + os.link(previous_cfile, cfile) + + except py_compile.PyCompileError as err: + success = False + if quiet >= 2: + return success + elif quiet: + print('*** Error compiling {!r}...'.format(fullname)) + else: + print('*** ', end='') + # escape non-printable characters in msg + msg = err.msg.encode(sys.stdout.encoding, + errors='backslashreplace') + msg = msg.decode(sys.stdout.encoding) + print(msg) + except (SyntaxError, UnicodeError, OSError) as e: + success = False + if quiet >= 2: + return success + elif quiet: + print('*** Error compiling {!r}...'.format(fullname)) + else: + print('*** ', end='') + print(e.__class__.__name__ + ':', e) + else: + if ok == 0: + success = False + return success + +def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=0, + legacy=False, optimize=-1, + invalidation_mode=None): + """Byte-compile all module on sys.path. + + Arguments (all optional): + + skip_curdir: if true, skip current directory (default True) + maxlevels: max recursion level (default 0) + force: as for compile_dir() (default False) + quiet: as for compile_dir() (default 0) + legacy: as for compile_dir() (default False) + optimize: as for compile_dir() (default -1) + invalidation_mode: as for compiler_dir() + """ + success = True + for dir in sys.path: + if (not dir or dir == os.curdir) and skip_curdir: + if quiet < 2: + print('Skipping current directory') + else: + success = success and compile_dir( + dir, + maxlevels, + None, + force, + quiet=quiet, + legacy=legacy, + optimize=optimize, + invalidation_mode=invalidation_mode, + ) + return success + + +def main(): + """Script main program.""" + import argparse + + parser = argparse.ArgumentParser( + description='Utilities to support installing Python libraries.') + parser.add_argument('-l', action='store_const', const=0, + default=None, dest='maxlevels', + help="don't recurse into subdirectories") + parser.add_argument('-r', type=int, dest='recursion', + help=('control the maximum recursion level. ' + 'if `-l` and `-r` options are specified, ' + 'then `-r` takes precedence.')) + parser.add_argument('-f', action='store_true', dest='force', + help='force rebuild even if timestamps are up to date') + parser.add_argument('-q', action='count', dest='quiet', default=0, + help='output only error messages; -qq will suppress ' + 'the error messages as well.') + parser.add_argument('-b', action='store_true', dest='legacy', + help='use legacy (pre-PEP3147) compiled file locations') + parser.add_argument('-d', metavar='DESTDIR', dest='ddir', default=None, + help=('directory to prepend to file paths for use in ' + 'compile-time tracebacks and in runtime ' + 'tracebacks in cases where the source file is ' + 'unavailable')) + parser.add_argument('-s', metavar='STRIPDIR', dest='stripdir', + default=None, + help=('part of path to left-strip from path ' + 'to source file - for example buildroot. ' + '`-d` and `-s` options cannot be ' + 'specified together.')) + parser.add_argument('-p', metavar='PREPENDDIR', dest='prependdir', + default=None, + help=('path to add as prefix to path ' + 'to source file - for example / to make ' + 'it absolute when some part is removed ' + 'by `-s` option. ' + '`-d` and `-p` options cannot be ' + 'specified together.')) + parser.add_argument('-x', metavar='REGEXP', dest='rx', default=None, + help=('skip files matching the regular expression; ' + 'the regexp is searched for in the full path ' + 'of each file considered for compilation')) + parser.add_argument('-i', metavar='FILE', dest='flist', + help=('add all the files and directories listed in ' + 'FILE to the list considered for compilation; ' + 'if "-", names are read from stdin')) + parser.add_argument('compile_dest', metavar='FILE|DIR', nargs='*', + help=('zero or more file and directory names ' + 'to compile; if no arguments given, defaults ' + 'to the equivalent of -l sys.path')) + parser.add_argument('-j', '--workers', default=1, + type=int, help='Run compileall concurrently') + parser.add_argument('-o', action='append', type=int, dest='opt_levels', + help=('Optimization levels to run compilation with. ' + 'Default is -1 which uses optimization level of ' + 'Python interpreter itself (specified by -O).')) + parser.add_argument('-e', metavar='DIR', dest='limit_sl_dest', + help='Ignore symlinks pointing outsite of the DIR') + parser.add_argument('--hardlink-dupes', action='store_true', + dest='hardlink_dupes', + help='Hardlink duplicated pyc files') + + if PY37: + invalidation_modes = [mode.name.lower().replace('_', '-') + for mode in py_compile.PycInvalidationMode] + parser.add_argument('--invalidation-mode', + choices=sorted(invalidation_modes), + help=('set .pyc invalidation mode; defaults to ' + '"checked-hash" if the SOURCE_DATE_EPOCH ' + 'environment variable is set, and ' + '"timestamp" otherwise.')) + + args = parser.parse_args() + compile_dests = args.compile_dest + + if args.rx: + import re + args.rx = re.compile(args.rx) + + if args.limit_sl_dest == "": + args.limit_sl_dest = None + + if args.recursion is not None: + maxlevels = args.recursion + else: + maxlevels = args.maxlevels + + if args.opt_levels is None: + args.opt_levels = [-1] + + if len(args.opt_levels) == 1 and args.hardlink_dupes: + parser.error(("Hardlinking of duplicated bytecode makes sense " + "only for more than one optimization level.")) + + if args.ddir is not None and ( + args.stripdir is not None or args.prependdir is not None + ): + parser.error("-d cannot be used in combination with -s or -p") + + # if flist is provided then load it + if args.flist: + try: + with (sys.stdin if args.flist=='-' else open(args.flist)) as f: + for line in f: + compile_dests.append(line.strip()) + except OSError: + if args.quiet < 2: + print("Error reading file list {}".format(args.flist)) + return False + + if args.workers is not None: + args.workers = args.workers or None + + if PY37 and args.invalidation_mode: + ivl_mode = args.invalidation_mode.replace('-', '_').upper() + invalidation_mode = py_compile.PycInvalidationMode[ivl_mode] + else: + invalidation_mode = None + + success = True + try: + if compile_dests: + for dest in compile_dests: + if os.path.isfile(dest): + if not compile_file(dest, args.ddir, args.force, args.rx, + args.quiet, args.legacy, + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir, + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): + success = False + else: + if not compile_dir(dest, maxlevels, args.ddir, + args.force, args.rx, args.quiet, + args.legacy, workers=args.workers, + invalidation_mode=invalidation_mode, + stripdir=args.stripdir, + prependdir=args.prependdir, + optimize=args.opt_levels, + limit_sl_dest=args.limit_sl_dest, + hardlink_dupes=args.hardlink_dupes): + success = False + return success + else: + return compile_path(legacy=args.legacy, force=args.force, + quiet=args.quiet, + invalidation_mode=invalidation_mode) + except KeyboardInterrupt: + if args.quiet < 2: + print("\n[interrupted]") + return False + return True + + +if __name__ == '__main__': + exit_status = int(not main()) + sys.exit(exit_status) diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index fc089c0..30d1557 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,14 +1,16 @@ Name: python-rpm-macros Version: 3 -Release: 45%{?dist} +Release: 46%{?dist} Summary: The unversioned Python RPM macros -License: MIT +# macros: MIT, compileall2.py: PSFv2 +License: MIT and Python Source0: macros.python Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -25,6 +27,7 @@ python?-devel packages require it. So install a python-devel package instead. %package -n python-srpm-macros Summary: RPM macros for building Python source packages +Requires: redhat-rpm-config %description -n python-srpm-macros RPM macros for building Python source packages. @@ -53,10 +56,13 @@ RPM macros for building Python 3 packages. %build %install -mkdir -p %{buildroot}/%{rpmmacrodir} +mkdir -p %{buildroot}%{rpmmacrodir} install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ - %{buildroot}/%{rpmmacrodir}/ + %{buildroot}%{rpmmacrodir}/ +mkdir -p %{buildroot}%{_rpmconfigdir}/redhat +install -m 644 %{SOURCE5} \ + %{buildroot}%{_rpmconfigdir}/redhat/ %files %{rpmmacrodir}/macros.python @@ -64,6 +70,7 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %files -n python-srpm-macros %{rpmmacrodir}/macros.python-srpm +%{_rpmconfigdir}/redhat/compileall2.py %files -n python2-rpm-macros %{rpmmacrodir}/macros.python2 @@ -73,6 +80,9 @@ install -m 644 %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} \ %changelog +* Mon Feb 10 2020 Miro Hrončok - 3-46 +- Add the compileall2 module (0.7.0) to be used in various Python spec files + * Fri Feb 07 2020 Miro Hrončok - 3-45 - Define %%py(2|3)?_shbang_opts_nodash to be used with pathfix.py -a From d479bad31946ca4c7ccf0efe04d1130e13554721 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 3 Mar 2020 15:26:45 +0100 Subject: [PATCH 6/6] Update of bundled compileall2 module to 0.7.1 (bugfix release) --- compileall2.py | 7 +++++++ python-rpm-macros.spec | 7 +++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compileall2.py b/compileall2.py index 8976fa0..c58e545 100644 --- a/compileall2.py +++ b/compileall2.py @@ -113,6 +113,13 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False, hardlink_dupes: hardlink duplicated pyc files """ ProcessPoolExecutor = None + if ddir is not None and (stripdir is not None or prependdir is not None): + raise ValueError(("Destination dir (ddir) cannot be used " + "in combination with stripdir or prependdir")) + if ddir is not None: + stripdir = dir + prependdir = ddir + ddir = None if workers is not None: if workers < 0: raise ValueError('workers must be greater or equal to 0') diff --git a/python-rpm-macros.spec b/python-rpm-macros.spec index 30d1557..2f7cd7a 100644 --- a/python-rpm-macros.spec +++ b/python-rpm-macros.spec @@ -1,6 +1,6 @@ Name: python-rpm-macros Version: 3 -Release: 46%{?dist} +Release: 47%{?dist} Summary: The unversioned Python RPM macros # macros: MIT, compileall2.py: PSFv2 @@ -10,7 +10,7 @@ Source1: macros.python-srpm Source2: macros.python2 Source3: macros.python3 Source4: macros.pybytecompile -Source5: https://github.com/fedora-python/compileall2/raw/v0.7.0/compileall2.py +Source5: https://github.com/fedora-python/compileall2/raw/v0.7.1/compileall2.py BuildArch: noarch # For %%python3_pkgversion used in %%python_provide @@ -80,6 +80,9 @@ install -m 644 %{SOURCE5} \ %changelog +* Tue Mar 31 2020 Lumír Balhar - 3-47 +- Update of bundled compileall2 module to 0.7.1 (bugfix release) + * Mon Feb 10 2020 Miro Hrončok - 3-46 - Add the compileall2 module (0.7.0) to be used in various Python spec files