Include pathfix.py in this package

This is in-line with https://fedoraproject.org/wiki/Changes/PythonSafePath#Opting_out

Backported from https://src.fedoraproject.org/rpms/python-rpm-macros/pull-request/141

macros.python

@@ -10,24 +10,20 @@
 %python_platform %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_platform())")
 %python_platform_triplet %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
 %python_ext_suffix %(RPM_BUILD_ROOT= %{__python} -Esc "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python_cache_tag %(RPM_BUILD_ROOT= %{__python} -Esc "import sys; print(sys.implementation.cache_tag)")
 
 %py_setup setup.py
-%py_shbang_opts -s
+%_py_shebang_s s
+%py_shbang_opts -%{?_py_shebang_s}
 %py_shbang_opts_nodash %(opts=%{py_shbang_opts}; echo ${opts#-})
 %py_shebang_flags %(opts=%{py_shbang_opts}; echo ${opts#-})
 %py_shebang_fix %{expand:\\\
-  if [ -f /usr/bin/pathfix%{python_version}.py ]; then
-    pathfix=/usr/bin/pathfix%{python_version}.py
-  else
-    # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly
-    pathfix=/usr/bin/pathfix.py
-  fi
   if [ -z "%{?py_shebang_flags}" ]; then
     shebang_flags="-k"
   else
     shebang_flags="-ka%{py_shebang_flags}"
   fi
-  $pathfix -pni %{__python} $shebang_flags}
+  %{__python} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python} $shebang_flags}
 
 # Use the slashes after expand so that the command starts on the same line as
 # the macro

macros.python3

@@ -8,24 +8,20 @@
 %python3_platform %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_platform())")
 %python3_platform_triplet %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('MULTIARCH'))")
 %python3_ext_suffix %(RPM_BUILD_ROOT= %{__python3} -Ic "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
+%python3_cache_tag %(RPM_BUILD_ROOT= %{__python3} -Ic "import sys; print(sys.implementation.cache_tag)")
 %py3dir %{_builddir}/python3-%{name}-%{version}-%{release}
 
-%py3_shbang_opts -s
+%_py3_shebang_s s
+%py3_shbang_opts -%{?_py3_shebang_s}
 %py3_shbang_opts_nodash %(opts=%{py3_shbang_opts}; echo ${opts#-})
 %py3_shebang_flags %(opts=%{py3_shbang_opts}; echo ${opts#-})
 %py3_shebang_fix %{expand:\\\
-  if [ -f /usr/bin/pathfix%{python3_version}.py ]; then
-    pathfix=/usr/bin/pathfix%{python3_version}.py
-  else
-    # older versions of Python don't have it and must BR /usr/bin/pathfix.py from python3-devel explicitly
-    pathfix=/usr/bin/pathfix.py
-  fi
   if [ -z "%{?py3_shebang_flags}" ]; then
     shebang_flags="-k"
   else
     shebang_flags="-ka%{py3_shebang_flags}"
   fi
-  $pathfix -pni %{__python3} $shebang_flags}
+  %{__python3} -B %{_rpmconfigdir}/redhat/pathfix.py -pni %{__python3} $shebang_flags}
 
 # Use the slashes after expand so that the command starts on the same line as
 # the macro
@@ -103,6 +99,7 @@
     pyminor = path:match("/python3.(%d+)/") or "*"
     dirname = path:match("(.*/)")
     modulename = path:match(".*/([^/]+).py")
+    -- %%python3_cache_tag is not used here because this macro supports not-installed CPythons
     print("\\n" .. dirname .. "__pycache__/" .. modulename .. ".cpython-3" .. pyminor .. "{,.opt-?}.pyc")
   end
 }

pathfix.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+from stat import *
+import getopt
+
+err = sys.stderr.write
+dbg = err
+rep = sys.stdout.write
+
+new_interpreter = None
+preserve_timestamps = False
+create_backup = True
+keep_flags = False
+add_flags = b''
+
+
+def main():
+    global new_interpreter
+    global preserve_timestamps
+    global create_backup
+    global keep_flags
+    global add_flags
+
+    usage = ('usage: %s -i /interpreter -p -n -k -a file-or-directory ...\n' %
+             sys.argv[0])
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'i:a:kpn')
+    except getopt.error as msg:
+        err(str(msg) + '\n')
+        err(usage)
+        sys.exit(2)
+    for o, a in opts:
+        if o == '-i':
+            new_interpreter = a.encode()
+        if o == '-p':
+            preserve_timestamps = True
+        if o == '-n':
+            create_backup = False
+        if o == '-k':
+            keep_flags = True
+        if o == '-a':
+            add_flags = a.encode()
+            if b' ' in add_flags:
+                err("-a option doesn't support whitespaces")
+                sys.exit(2)
+    if not new_interpreter or not new_interpreter.startswith(b'/') or \
+           not args:
+        err('-i option or file-or-directory missing\n')
+        err(usage)
+        sys.exit(2)
+    bad = 0
+    for arg in args:
+        if os.path.isdir(arg):
+            if recursedown(arg): bad = 1
+        elif os.path.islink(arg):
+            err(arg + ': will not process symbolic links\n')
+            bad = 1
+        else:
+            if fix(arg): bad = 1
+    sys.exit(bad)
+
+
+def ispython(name):
+    return name.endswith('.py')
+
+
+def recursedown(dirname):
+    dbg('recursedown(%r)\n' % (dirname,))
+    bad = 0
+    try:
+        names = os.listdir(dirname)
+    except OSError as msg:
+        err('%s: cannot list directory: %r\n' % (dirname, msg))
+        return 1
+    names.sort()
+    subdirs = []
+    for name in names:
+        if name in (os.curdir, os.pardir): continue
+        fullname = os.path.join(dirname, name)
+        if os.path.islink(fullname): pass
+        elif os.path.isdir(fullname):
+            subdirs.append(fullname)
+        elif ispython(name):
+            if fix(fullname): bad = 1
+    for fullname in subdirs:
+        if recursedown(fullname): bad = 1
+    return bad
+
+
+def fix(filename):
+##  dbg('fix(%r)\n' % (filename,))
+    try:
+        f = open(filename, 'rb')
+    except IOError as msg:
+        err('%s: cannot open: %r\n' % (filename, msg))
+        return 1
+    with f:
+        line = f.readline()
+        fixed = fixline(line)
+        if line == fixed:
+            rep(filename+': no change\n')
+            return
+        head, tail = os.path.split(filename)
+        tempname = os.path.join(head, '@' + tail)
+        try:
+            g = open(tempname, 'wb')
+        except IOError as msg:
+            err('%s: cannot create: %r\n' % (tempname, msg))
+            return 1
+        with g:
+            rep(filename + ': updating\n')
+            g.write(fixed)
+            BUFSIZE = 8*1024
+            while 1:
+                buf = f.read(BUFSIZE)
+                if not buf: break
+                g.write(buf)
+
+    # Finishing touch -- move files
+
+    mtime = None
+    atime = None
+    # First copy the file's mode to the temp file
+    try:
+        statbuf = os.stat(filename)
+        mtime = statbuf.st_mtime
+        atime = statbuf.st_atime
+        os.chmod(tempname, statbuf[ST_MODE] & 0o7777)
+    except OSError as msg:
+        err('%s: warning: chmod failed (%r)\n' % (tempname, msg))
+    # Then make a backup of the original file as filename~
+    if create_backup:
+        try:
+            os.rename(filename, filename + '~')
+        except OSError as msg:
+            err('%s: warning: backup failed (%r)\n' % (filename, msg))
+    else:
+        try:
+            os.remove(filename)
+        except OSError as msg:
+            err('%s: warning: removing failed (%r)\n' % (filename, msg))
+    # Now move the temp file to the original file
+    try:
+        os.rename(tempname, filename)
+    except OSError as msg:
+        err('%s: rename failed (%r)\n' % (filename, msg))
+        return 1
+    if preserve_timestamps:
+        if atime and mtime:
+            try:
+                os.utime(filename, (atime, mtime))
+            except OSError as msg:
+                err('%s: reset of timestamp failed (%r)\n' % (filename, msg))
+                return 1
+    # Return success
+    return 0
+
+
+def parse_shebang(shebangline):
+    shebangline = shebangline.rstrip(b'\n')
+    start = shebangline.find(b' -')
+    if start == -1:
+        return b''
+    return shebangline[start:]
+
+
+def populate_flags(shebangline):
+    old_flags = b''
+    if keep_flags:
+        old_flags = parse_shebang(shebangline)
+        if old_flags:
+            old_flags = old_flags[2:]
+    if not (old_flags or add_flags):
+        return b''
+    # On Linux, the entire string following the interpreter name
+    # is passed as a single argument to the interpreter.
+    # e.g. "#! /usr/bin/python3 -W Error -s" runs "/usr/bin/python3 "-W Error -s"
+    # so shebang should have single '-' where flags are given and
+    # flag might need argument for that reasons adding new flags is
+    # between '-' and original flags
+    # e.g. #! /usr/bin/python3 -sW Error
+    return b' -' + add_flags + old_flags
+
+
+def fixline(line):
+    if not line.startswith(b'#!'):
+        return line
+
+    if b"python" not in line:
+        return line
+
+    flags = populate_flags(line)
+    return b'#! ' + new_interpreter + flags + b'\n'
+
+
+if __name__ == '__main__':
+    main()

python-rpm-macros.spec

@@ -16,6 +16,8 @@ Source201:      python.lua
 %global compileall2_version 0.7.1
 Source301:      https://github.com/fedora-python/compileall2/raw/v%{compileall2_version}/compileall2.py
 Source302:      import_all_modules.py
+%global pathfix_version 1.0.0
+Source303:      https://github.com/fedora-python/pathfix/raw/v%{pathfix_version}/pathfix.py
 
 # BRP scripts
 # This one is from redhat-rpm-config < 190
@@ -34,6 +36,7 @@ Source403:      brp-fix-pyc-reproducibility
 # macros and lua: MIT
 # import_all_modules.py: MIT
 # compileall2.py: PSFv2
+# pathfix.py: PSFv2
 # brp scripts: GPLv2+
 License:        MIT and Python and GPLv2+
 
@@ -49,7 +52,7 @@ elseif posix.stat('macros.python-srpm') then
 end
 }
 Version:        %{__default_python3_version}
-Release:        18%{?dist}
+Release:        20%{?dist}
 
 BuildArch:      noarch
 
@@ -103,6 +106,10 @@ RPM macros for building Python 3 packages.
 %autosetup -c -T
 cp -a %{sources} .
 
+# We want to have shebang in the script upstream but not here so
+# the package with macros does not depend on Python.
+sed -i '1s=^#!/usr/bin/env python3==' pathfix.py
+
 
 %install
 mkdir -p %{buildroot}%{rpmmacrodir}
@@ -114,7 +121,7 @@ install -p -m 644 -t %{buildroot}%{_rpmluadir}/fedora/srpm python.lua
 mkdir -p %{buildroot}%{_rpmconfigdir}/redhat
 install -m 644 compileall2.py %{buildroot}%{_rpmconfigdir}/redhat/
 install -m 644 import_all_modules.py %{buildroot}%{_rpmconfigdir}/redhat/
-
+install -m 644 pathfix.py %{buildroot}%{_rpmconfigdir}/redhat/
 install -m 755 brp-* %{buildroot}%{_rpmconfigdir}/redhat/
 
 
@@ -137,6 +144,7 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true
 %{rpmmacrodir}/macros.python
 %{rpmmacrodir}/macros.pybytecompile
 %{_rpmconfigdir}/redhat/import_all_modules.py
+%{_rpmconfigdir}/redhat/pathfix.py
 
 %files -n python-srpm-macros
 %{rpmmacrodir}/macros.python-srpm
@@ -151,6 +159,13 @@ grep -E '^#[^%%]*%%[^%%]' %{buildroot}%{rpmmacrodir}/macros.* && exit 1 || true
 
 
 %changelog
+* Tue Oct 25 2022 Lumír Balhar <lbalhar@redhat.com> - 3.10-20
+- Include pathfix.py in this package
+
+* Tue Jul 19 2022 Miro Hrončok <mhroncok@redhat.com> - 3.10-19
+- Define %%python3_cache_tag / %%python_cache_tag, e.g. cpython-311
+- Define and use %%{_py3_shebang_s} in the shebang macros for easier opt-out
+
 * Thu May 26 2022 Owen Taylor <otaylor@redhat.com> - 3.10-18
 - Support installing to %%{_prefix} other than /usr
 

tests/test_evals.py

@@ -72,8 +72,17 @@ def get_alt_x_y():
         raise ValueError(f"${env_name} must be X.Y")
     return alternate_python_version
 
-# We don't use the decorator, to be able to call the function itselef
+
+def get_alt_xy():
+    """
+    Same as get_alt_x_y() but without a dot
+    """
+    return get_alt_x_y().replace(".", "")
+
+
+# We don't use decorators, to be able to call the functions directly
 alt_x_y = pytest.fixture(scope="session")(get_alt_x_y)
+alt_xy = pytest.fixture(scope="session")(get_alt_xy)
 
 
 def shell_stdout(script):
@@ -394,7 +403,7 @@ def test_pypi_source_explicit_tilde():
 
 def test_py3_shebang_fix():
     cmd = rpm_eval('%py3_shebang_fix arg1 arg2 arg3')[-1].strip()
-    assert cmd == '$pathfix -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3'
+    assert cmd == '/usr/bin/python3 -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/python3 $shebang_flags arg1 arg2 arg3'
 
 
 def test_py3_shebang_fix_default_shebang_flags():
@@ -409,6 +418,13 @@ def test_py3_shebang_fix_custom_shebang_flags():
     assert shell_stdout('\n'.join(lines)) == '-kaEs'
 
 
+@pytest.mark.parametrize('_py3_shebang_s', [None, '%{nil}'])
+def test_py3_shebang_fix_undefined_py3_shebang_s(_py3_shebang_s):
+    lines = rpm_eval('%py3_shebang_fix arg1 arg2', _py3_shebang_s=_py3_shebang_s)
+    lines[-1] = 'echo $shebang_flags'
+    assert shell_stdout('\n'.join(lines)) == '-k'
+
+
 @pytest.mark.parametrize('flags', [None, '%{nil}'])
 def test_py3_shebang_fix_no_shebang_flags(flags):
     lines = rpm_eval('%py3_shebang_fix arg1 arg2', py3_shebang_flags=flags)
@@ -418,7 +434,7 @@ def test_py3_shebang_fix_no_shebang_flags(flags):
 
 def test_py_shebang_fix_custom_python():
     cmd = rpm_eval('%py_shebang_fix arg1 arg2 arg3', __python='/usr/bin/pypy')[-1].strip()
-    assert cmd == '$pathfix -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3'
+    assert cmd == '/usr/bin/pypy -B /usr/lib/rpm/redhat/pathfix.py -pni /usr/bin/pypy $shebang_flags arg1 arg2 arg3'
 
 
 def test_pycached_in_sitelib():
@@ -638,6 +654,7 @@ unversioned_macros = pytest.mark.parametrize('macro', [
     '%python_platform',
     '%python_platform_triplet',
     '%python_ext_suffix',
+    '%python_cache_tag',
     '%py_shebang_fix',
     '%py_build',
     '%py_build_egg',
@@ -676,6 +693,18 @@ def test_ext_suffix():
     assert rpm_eval("%python3_ext_suffix") == [f".cpython-{XY}-x86_64-linux-gnu.so"]
 
 
+def test_cache_tag():
+    assert rpm_eval("%python3_cache_tag") == [f"cpython-{XY}"]
+
+
+def test_cache_tag_alternate_python(alt_x_y, alt_xy):
+    assert rpm_eval("%python_cache_tag", __python=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"]
+
+
+def test_cache_tag_alternate_python3(alt_x_y, alt_xy):
+    assert rpm_eval("%python3_cache_tag", __python3=f"/usr/bin/python{alt_x_y}") == [f"cpython-{alt_xy}"]
+
+
 def test_python_sitelib_value_python3():
     macro = '%python_sitelib'
     assert rpm_eval(macro, __python='%__python3') == [f'/usr/lib/python{X_Y}/site-packages']