diff --git a/Makefile.rhelver b/Makefile.rhelver index 3fa0703ab..6476a6464 100644 --- a/Makefile.rhelver +++ b/Makefile.rhelver @@ -12,7 +12,7 @@ RHEL_MINOR = 99 # # Use this spot to avoid future merge conflicts. # Do not trim this comment. -RHEL_RELEASE = 23 +RHEL_RELEASE = 24 # # RHEL_REBASE_NUM diff --git a/kernel-s390x-debug-rhel.config b/kernel-s390x-debug-rhel.config index a66aa0e6b..1b7d049a5 100644 --- a/kernel-s390x-debug-rhel.config +++ b/kernel-s390x-debug-rhel.config @@ -5206,8 +5206,8 @@ CONFIG_REALTEK_PHY_HWMON=y # CONFIG_REDMI_WMI is not set # CONFIG_REED_SOLOMON_TEST is not set # CONFIG_REGMAP_BUILD is not set -# CONFIG_REGMAP is not set CONFIG_REGMAP_KUNIT=m +CONFIG_REGMAP=y # CONFIG_REGULATOR_88PG86X is not set # CONFIG_REGULATOR_ACT8865 is not set # CONFIG_REGULATOR_AD5398 is not set diff --git a/kernel-s390x-rhel.config b/kernel-s390x-rhel.config index 0a25d39d9..8e5e0f220 100644 --- a/kernel-s390x-rhel.config +++ b/kernel-s390x-rhel.config @@ -5182,8 +5182,8 @@ CONFIG_REALTEK_PHY_HWMON=y # CONFIG_REDMI_WMI is not set # CONFIG_REED_SOLOMON_TEST is not set # CONFIG_REGMAP_BUILD is not set -# CONFIG_REGMAP is not set CONFIG_REGMAP_KUNIT=m +CONFIG_REGMAP=y # CONFIG_REGULATOR_88PG86X is not set # CONFIG_REGULATOR_ACT8865 is not set # CONFIG_REGULATOR_AD5398 is not set diff --git a/kernel-s390x-zfcpdump-rhel.config b/kernel-s390x-zfcpdump-rhel.config index fc0ed4eda..9a7afc489 100644 --- a/kernel-s390x-zfcpdump-rhel.config +++ b/kernel-s390x-zfcpdump-rhel.config @@ -5194,8 +5194,8 @@ CONFIG_REALTEK_PHY_HWMON=y # CONFIG_REDMI_WMI is not set # CONFIG_REED_SOLOMON_TEST is not set # CONFIG_REGMAP_BUILD is not set -# CONFIG_REGMAP is not set CONFIG_REGMAP_KUNIT=m +CONFIG_REGMAP=y # CONFIG_REGULATOR_88PG86X is not set # CONFIG_REGULATOR_ACT8865 is not set # CONFIG_REGULATOR_AD5398 is not set diff --git a/kernel.changelog b/kernel.changelog index 79a78b1ac..cf6348b92 100644 --- a/kernel.changelog +++ b/kernel.changelog @@ -1,8 +1,14 @@ -* Thu Jul 09 2026 Fedora Kernel Team [7.2.0-0.rc2.0e35b9b6ec0f.23] +* Sat Jul 11 2026 Fedora Kernel Team [7.2.0-0.rc2.dd3210c47e8d.24] - redhat/kernel.spec: require libbabeltrace2-devel (Yaakov Selkowitz) - automotive: enable HUGETLBFS to workaround build error (Scott Weaver) Resolves: +* Sat Jul 11 2026 Fedora Kernel Team [7.2.0-0.rc2.dd3210c47e8d.23] +- Turn on CONFIG_REGMAP for RHEL s390x pending for mismatch (Justin M. Forbes) +- redhat: add kmap.py tool and kernel-kmap-internal package (Rado Vrbovsky) +- Linux v7.2.0-0.rc2.dd3210c47e8d +Resolves: + * Thu Jul 09 2026 Fedora Kernel Team [7.2.0-0.rc2.0e35b9b6ec0f.22] - uki-virt: Add fdisk to the initramfs to make systemd-repart functional (Vitaly Kuznetsov) - redhat/configs: Enable Epson RX8111 RTC driver for arm/aarch64 builds. Signed-off-by: Nirmala Dalvi (Nirmala Dalvi) diff --git a/kernel.spec b/kernel.spec index 0cea8f7dc..42a3af04b 100644 --- a/kernel.spec +++ b/kernel.spec @@ -190,13 +190,13 @@ Summary: The Linux kernel %define specrpmversion 7.2.0 %define specversion 7.2.0 %define patchversion 7.2 -%define pkgrelease 0.rc2.260709g0e35b9b6ec0f.23 +%define pkgrelease 0.rc2.260711gdd3210c47e8d.24 %define kversion 7 -%define tarfile_release 7.2-rc2-22-g0e35b9b6ec0f +%define tarfile_release 7.2-rc2-264-gdd3210c47e8d # This is needed to do merge window version magic %define patchlevel 2 # This allows pkg_release to have configurable %%{?dist} tag -%define specrelease 0.rc2.260709g0e35b9b6ec0f.23%{?buildid}%{?dist} +%define specrelease 0.rc2.260711gdd3210c47e8d.24%{?buildid}%{?dist} # This defines the kabi tarball version %define kabiversion 7.2.0 @@ -282,6 +282,8 @@ Summary: The Linux kernel %define with_ynl %{?_without_ynl: 0} %{?!_without_ynl: 1} # kernel-debuginfo %define with_debuginfo %{?_without_debuginfo: 0} %{?!_without_debuginfo: 1} +# kernel-kmap-internal: source-to-module mapping data (JSON) +%define with_kmap %{?_without_kmap: 0} %{?!_without_kmap: 1} # kernel-abi-stablelists %define with_kernel_abi_stablelists %{?_without_kernel_abi_stablelists: 0} %{?!_without_kernel_abi_stablelists: 1} # internal samples and selftests @@ -705,6 +707,10 @@ Summary: The Linux kernel %define _enable_debug_packages 0 %endif +%ifnarch x86_64 ppc64le s390x aarch64 riscv64 +%define with_kmap 0 +%endif + # Architectures we build tools/cpupower on %if 0%{?fedora} %define cpupowerarchs %{ix86} x86_64 ppc64le aarch64 riscv64 @@ -1204,6 +1210,9 @@ Source491: %{name}-x86_64-automotive-debug-rhel.config # Sources for kernel-tools Source2002: kvm_stat.logrotate +# Sources for kernel-kmap-internal +Source2100: kmap.py + # Some people enjoy building customized kernels from the dist-git in Fedora and # use this to override configuration options. One day they may all use the # source tree, but in the mean time we carry this to support the legacy workflow @@ -1450,7 +1459,7 @@ This package provides debug information for package %{name}-tools. # symlinks because of the trailing nonmatching alternation and # the leading .*, because of find-debuginfo.sh's buggy handling # of matching the pattern against the symlinks file. -%{expand:%%global _find_debuginfo_opts %{?_find_debuginfo_opts} -p '.*%%{_bindir}/bootconfig(\.debug)?|.*%%{_bindir}/centrino-decode(\.debug)?|.*%%{_bindir}/powernow-k8-decode(\.debug)?|.*%%{_bindir}/cpupower(\.debug)?|.*%%{_libdir}/libcpupower.*|.*%%{python3_sitearch}/_raw_pylibcpupower.*|.*%%{_bindir}/turbostat(\.debug)?|.*%%{_bindir}/x86_energy_perf_policy(\.debug)?|.*%%{_bindir}/tmon(\.debug)?|.*%%{_bindir}/lsgpio(\.debug)?|.*%%{_bindir}/gpio-hammer(\.debug)?|.*%%{_bindir}/gpio-event-mon(\.debug)?|.*%%{_bindir}/gpio-watch(\.debug)?|.*%%{_bindir}/iio_event_monitor(\.debug)?|.*%%{_bindir}/iio_generic_buffer(\.debug)?|.*%%{_bindir}/lsiio(\.debug)?|.*%%{_bindir}/intel-speed-select(\.debug)?|.*%%{_bindir}/page_owner_sort(\.debug)?|.*%%{_bindir}/slabinfo(\.debug)?|.*%%{_sbindir}/intel_sdsi(\.debug)?|XXX' -o %{name}-tools-debuginfo.list} +%{expand:%%global _find_debuginfo_opts %{?_find_debuginfo_opts} -p '.*%%{_bindir}/bootconfig(\.debug)?|.*%%{_bindir}/centrino-decode(\.debug)?|.*%%{_bindir}/powernow-k8-decode(\.debug)?|.*%%{_bindir}/cpupower(\.debug)?|.*%%{_libdir}/libcpupower.*|.*%%{python3_sitearch}/_raw_pylibcpupower.*|.*%%{_bindir}/turbostat(\.debug)?|.*%%{_bindir}/x86_energy_perf_policy(\.debug)?|.*%%{_bindir}/tmon(\.debug)?|.*%%{_bindir}/lsgpio(\.debug)?|.*%%{_bindir}/gpio-hammer(\.debug)?|.*%%{_bindir}/gpio-event-mon(\.debug)?|.*%%{_bindir}/gpio-watch(\.debug)?|.*%%{_bindir}/iio_event_monitor(\.debug)?|.*%%{_bindir}/iio_generic_buffer(\.debug)?|.*%%{_bindir}/lsiio(\.debug)?|.*%%{_bindir}/intel-speed-select(\.debug)?|.*%%{_bindir}/page_owner_sort(\.debug)?|.*%%{_bindir}/slabinfo(\.debug)?|.*%%{_sbindir}/intel_sdsi(\.debug)?|.*%%{_bindir}/ynltool(\.debug)?|XXX' -o %{name}-tools-debuginfo.list} %if %{with_tools} && %{with_ynl} %package -n python3-%{name}-tools @@ -1529,6 +1538,27 @@ This package provides debug information for the rv package. # with_tools %endif +%if %{with_kmap} && %{with_base} +%package -n %{name}-kmap-internal +Summary: Kernel source-to-module mapping data and module list +Group: Development/System +BuildRequires: python3 +%description -n %{name}-kmap-internal +The %{name}-kmap-internal package contains a JSON mapping file that describes +which C source files contributed to each kernel module and to the built-in +vmlinux image, and which RPM package each module is shipped in. + +Files installed under /usr/share/%{name}-kmap-internal/: + kernel-map-.json - unified mapping file for all kernel variants containing: + - variants: list of variant names (e.g., ["stock", "rt", "automotive"]) + - source-map: source file mappings + - obj-src: maps kernel objects to source files with variant indices + - src-obj: maps source files to kernel objects with variant indices + - module-map: module to RPM mappings + - module-rpm: maps module names to RPM package names with variant indices + - rpm-modules: maps RPM package names to module names with variant indices +%endif + %if %{with_selftests} %package selftests-internal @@ -3183,6 +3213,44 @@ BuildKernel() { %endif fi # $DoModules -eq 1 +%if %{with_kmap} && %{with_base} + # Generate source-to-module mapping data using kmap.py. + # Must run before the next BuildKernel call issues make mrproper, which + # would wipe the .cmd files that kmap.py reads. Skip debug variants + # (same source mapping as the base kernel). + if [[ "$Variant" != *debug* ]]; then + _kmap_bdir=$(pwd) + _kmap_basedir=%{_builddir}/kmap-data + _kmap_merged=$_kmap_basedir/kernel-map.json + _kmap_variant=${Variant:-stock} + _kmap_listprefix=../kernel${Variant:+-${Variant}} + mkdir -p $_kmap_basedir + + # Build kmap.py arguments; merge with existing data if present + _kmap_args="--directory $_kmap_bdir --outputdir $_kmap_basedir --rhel upstream --variant $_kmap_variant --time" + _kmap_args="$_kmap_args --vmlinux-rpm %{name}-core-%{KVERREL}.rpm" + if [ -f "$_kmap_merged" ]; then + _kmap_args="$_kmap_args --input kernel-map.json" + fi + + # Add module list files for module-to-RPM mapping + for _kmap_type in modules-core modules modules-extra modules-internal; do + if [ -f "${_kmap_listprefix}-${_kmap_type}.list" ]; then + _kmap_rpm=%{name}-${_kmap_type}-%{KVERREL}.rpm + _kmap_args="$_kmap_args --module-list ${_kmap_rpm}:${_kmap_listprefix}-${_kmap_type}.list" + fi + done +%if 0%{!?fedora:1} + if [ -f "${_kmap_listprefix}-modules-partner.list" ]; then + _kmap_rpm=%{name}-modules-partner-%{KVERREL}.rpm + _kmap_args="$_kmap_args --module-list ${_kmap_rpm}:${_kmap_listprefix}-modules-partner.list" + fi +%endif + + python3 %{SOURCE2100} $_kmap_args + fi +%endif + remove_depmod_files() { # remove files that will be auto generated by depmod at rpm -i time @@ -4091,6 +4159,14 @@ find -type f ! -executable -exec install -D -m644 {} %{buildroot}%{_libexecdir}/ popd %endif +%if %{with_kmap} && %{with_base} +# Install kmap data files produced during %build. +_kmap_basedir=%{_builddir}/kmap-data +_kmap_destbase=$RPM_BUILD_ROOT%{_datadir}/%{name}-kmap-internal +mkdir -p $_kmap_destbase +install -m 644 $_kmap_basedir/kernel-map.json $_kmap_destbase/kernel-map-%{KVERREL}.json +%endif + ### ### clean ### @@ -4649,6 +4725,13 @@ fi\ %{_libexecdir}/kselftests %endif +%if %{with_kmap} && %{with_base} +%files -n %{name}-kmap-internal +%defattr(-,root,root) +%dir %{_datadir}/%{name}-kmap-internal +%{_datadir}/%{name}-kmap-internal/kernel-map-%{KVERREL}.json +%endif + # empty meta-package %if %{with_stock_base} %ifnarch %nobuildarches noarch @@ -4859,10 +4942,18 @@ fi\ # # %changelog -* Thu Jul 09 2026 Fedora Kernel Team [7.2.0-0.rc2.0e35b9b6ec0f.23] +* Sat Jul 11 2026 Justin M. Forbes [7.2.0-0.rc2.260711gdd3210c47e8d.24] +- redhat: move ynltool debuginfo to kernel-tools-debuginfo (Augusto Caringi) + +* Sat Jul 11 2026 Fedora Kernel Team [7.2.0-0.rc2.dd3210c47e8d.24] - redhat/kernel.spec: require libbabeltrace2-devel (Yaakov Selkowitz) - automotive: enable HUGETLBFS to workaround build error (Scott Weaver) +* Sat Jul 11 2026 Fedora Kernel Team [7.2.0-0.rc2.dd3210c47e8d.23] +- Turn on CONFIG_REGMAP for RHEL s390x pending for mismatch (Justin M. Forbes) +- redhat: add kmap.py tool and kernel-kmap-internal package (Rado Vrbovsky) +- Linux v7.2.0-0.rc2.dd3210c47e8d + * Thu Jul 09 2026 Fedora Kernel Team [7.2.0-0.rc2.0e35b9b6ec0f.22] - uki-virt: Add fdisk to the initramfs to make systemd-repart functional (Vitaly Kuznetsov) - redhat/configs: Enable Epson RX8111 RTC driver for arm/aarch64 builds. Signed-off-by: Nirmala Dalvi (Nirmala Dalvi) diff --git a/kmap.py b/kmap.py new file mode 100755 index 000000000..70af504d1 --- /dev/null +++ b/kmap.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python3 + +""" +kmap.py - Kernel Source-to-Binary Mapping Tool + +DESCRIPTION + This tool parses Linux kernel build artifacts (.cmd files, archives) to + create a JSON mapping of which source files contribute to which kernel + binaries (modules and vmlinux). It supports merging data from multiple + kernel build variants (e.g., stock, debug, rt) into a single output + file using variant indices to track per-variant mappings. + + Supported RHEL versions: 7, 8, 9, upstream + +FEATURES + - Parses .cmd files generated by kbuild to extract compilation commands + - Supports C (gcc/clang) and Rust (rustc) source files + - Handles built-in objects via built-in.o.cmd (RHEL 7), built-in.a + (RHEL 8/9), or vmlinux.a (upstream) + - Merges multiple kernel variants into single output with variant indices + - Maps modules to their containing RPM packages + +OUTPUT FORMAT + The output is a JSON file with the following structure: + + { + "variants": ["stock", "rt", ...], + "source-map": { + "obj-src": {: {: []}}, + "src-obj": {: {: []}} + }, + "module-map": { + "module-rpm": {: {: []}}, + "rpm-modules": {: {: []}} + } + } + +PARSING THE OUTPUT + import json + + with open('kernel-map.json') as f: + data = json.load(f) + + variants = data['variants'] + obj_src = data['source-map']['obj-src'] + src_obj = data['source-map']['src-obj'] + module_rpm = data['module-map']['module-rpm'] + rpm_modules = data['module-map']['rpm-modules'] + + # Get sources for vmlinux in variant "stock" (index 0) + vmlinux_sources = [src for src, indices in obj_src['vmlinux'].items() + if 0 in indices] + + # Find which object a source contributes to + fork_objects = [obj for obj, indices in src_obj['kernel/fork.c'].items() + if 0 in indices] + + # Get RPM for a module + def get_rpm(module, variant_idx): + for rpm, indices in module_rpm.get(module, {}).items(): + if variant_idx in indices: + return rpm + return None + +USAGE + # Single variant + ./kmap.py -d /build/stock -r upstream -v stock -o /output + + # Merge multiple variants + ./kmap.py -d /build/stock -r upstream -v stock -o /output + ./kmap.py -d /build/debug -r upstream -v debug -o /output -i kernel-map.json + ./kmap.py -d /build/rt -r upstream -v rt -o /output -i kernel-map.json + + # With module-to-RPM mapping + ./kmap.py -d /build -r upstream -v stock -o /output \\ + --vmlinux-rpm kernel-core-5.14.0.rpm \\ + --module-list kernel-modules-5.14.0.rpm:modules.list + +CREDITS + Inspired by gen_compile_commands.py and Joe Lawrence. + Time measurement option added for Jan Stancek. +""" + +import re +import os +import pathlib +import json +import argparse +import subprocess +import time + + +# ============================================================================= +# Constants +# ============================================================================= + +VMLINUX = 'vmlinux' +KERNEL_PREFIX = 'kernel/' + +# Supported compilers and linkers +COMPILERS = ('gcc', 'clang') +LINKER = 'ld' +RUST_COMPILER = 'rustc' + + +# ============================================================================= +# Regex Patterns +# +# These patterns parse the kernel build .cmd files to extract: +# - What compiler/linker was used +# - What source files were compiled +# - What object files were linked +# ============================================================================= + +# Pattern for .ko.cmd files (module linking) +# Matches: cmd_path/to/module.ko := ld -o +# Note: upstream kernels use 'savedcmd_' prefix instead of 'cmd_' +_KO_CMD_PATTERN = ( + r'^(saved)?cmd_[^ ]*\.ko\s*:=\s*ld\s+' + r'(?P.*) -o (?P\S+) (?P[^;]+?)\s*(?:;|$)' +) + +# Pattern for .o.cmd files (object compilation) +# Matches multiple formats: +# - C/asm: cmd_*.o := gcc/clang -o +# - Rust: cmd_*.o := rustc --emit=obj= +# - Rust source mapping: source_*.o := +_O_CMD_PATTERN = ( + r'^(saved)?(?:cmd_|source_)[^ ]*\.o\s+:=\s+' + r'(?:' + r'(?P(?:\S+=\S+\s+)*)?' + r'(?Pgcc|clang|ld|rustc)\s+' + r'(?P.*?)' + r'(?:' + r'\s+-o\s+(?P\S+)\s+(?P[^;]+?)' + r'|' + r'\s+--emit=obj=(?P\S+).*?\s+(?P\S+\.rs)' + r')' + r'|' + r'(?P\S+\.rs)' + r')\s*(?:;|$)' +) + +ko_matcher = re.compile(_KO_CMD_PATTERN) +o_matcher = re.compile(_O_CMD_PATTERN) + + +# ============================================================================= +# Utility Functions +# ============================================================================= + +def parse_args(): + """ + Parse command line arguments + """ + parser = argparse.ArgumentParser( + prog='Kernel Build Mapper', + description='Scrape Linux Kernel .cmd files to map contribution ' + 'of individual source files to kernel binaries.', + ) + + parser.add_argument('-d', '--directory', + help='Directory with kernel build artifacts.', + default='', + type=str) + + parser.add_argument('-o', '--outputdir', + help='Directory where to put generated files.', + default='', + type=str) + + parser.add_argument('-p', '--prefix', + help='Name prefix to the generated files.', + default='', + type=str) + + parser.add_argument('-r', '--rhel', + help='Specify to what version of RHEL artifacts belong (upstream for RHEL 10+/kernel-ark)', + choices=['7', '8', '9', 'upstream'], + required=True) + + parser.add_argument('-m', '--no-modules', + help='*SKIP* collecting information about modules', + action='store_true') + + parser.add_argument('-b', '--no-builtin', + help='*SKIP* collecting information about built-in', + action='store_true') + + parser.add_argument('-t', '--time', + help='Print elapsed time at the end', + action='store_true') + + parser.add_argument('-v', '--variant', + help='Name of this kernel variant (e.g., stock, debug, rt)', + required=True, + type=str) + + parser.add_argument('-i', '--input', + help='Existing JSON file to merge with (from previous variant runs)', + default='', + type=str) + + parser.add_argument('--module-list', + help='Module list in format : (can be specified multiple times)', + action='append', + default=[], + metavar='RPM:PATH') + + parser.add_argument('--vmlinux-rpm', + help='RPM package name for vmlinux (e.g., "kernel-5.14.0.rpm")', + default='', + type=str) + + return parser.parse_args() + + +def load_file(filename, build_dir=''): + """ + Load a text file and return non-empty lines as a list + """ + full_path = os.path.join(build_dir, filename) + + try: + with open(full_path, 'rt', encoding='utf-8') as file: + return [line.rstrip() for line in file if line.rstrip()] + except FileNotFoundError: + print(f'File "{full_path}" not found.') + return [] + + +def name_to_cmd(filename, suffix): + """ + Convert object filename to corresponding .cmd script filename + + Example: kernel/fork.o -> kernel/.fork.o.cmd + """ + path, _ = os.path.splitext(filename) + dirname = os.path.dirname(path) + basename = os.path.basename(path) + + return os.path.join(dirname, '.' + basename + suffix) + + +def strip_path_prefixes(path, build_dir): + """ + Normalize path by removing build directory prefix and other artifacts + """ + path = pathlib.Path(path) + + if build_dir: + try: + return str(path.resolve().relative_to(pathlib.Path(build_dir).resolve())) + except ValueError: + pass + + if path.is_absolute(): + path = pathlib.Path(*path.parts[1:]) + + while path.parts and path.parts[0] == 'repos': + path = pathlib.Path(*path.parts[1:]) + + return str(path) + + +def is_external_source(path): + """ + Check if path should be excluded from source mapping + + Filters out: + - Rust toolchain sources (not part of kernel source) + - Generated assembly files (e.g., initramfs_data.S) + """ + if 'rustlib' in path: + return True + if path.endswith('_data.S'): + return True + return False + + +def is_mod_file_reference(ld_input): + """ + Check if linker input is a .mod file reference (RHEL 9+ module linking) + + In RHEL 9+, module linking uses @path/to/module.mod files that contain + the list of object files to link. + """ + files = ld_input.split() + return len(files) == 1 and files[0].startswith('@') and files[0].endswith('.mod') + + +# ============================================================================= +# SourceMap Class +# ============================================================================= + +class SourceMap: + """ + Collects and manages kernel source-to-binary mappings. + + This class parses kernel build artifacts (.cmd files, archives) to determine + which source files contribute to which kernel binaries (modules and vmlinux). + + Data structures: + obj_to_src: {object_name: {source_file: [variant_indices]}} + src_to_obj: {source_file: {object_name: [variant_indices]}} + module_rpm: {module_name: {rpm_name: [variant_indices]}} + rpm_to_modules: {rpm_name: {module_name: [variant_indices]}} + + Variant indices correspond to positions in the 'variants' list. + """ + + # Dispatch table for RHEL-specific builtin collection methods + BUILTIN_COLLECTORS = { + '7': '_collect_builtin_rhel7', + '8': '_collect_builtin_archive', + '9': '_collect_builtin_archive', + 'upstream': '_collect_builtin_vmlinux_archive', + } + + def __init__(self, build_dir='', output_dir='', outprefix='', rhel='7', variant=''): + # Source mappings + self.obj_to_src = {} + self.src_to_obj = {} + # Module-to-RPM mappings + self.module_rpm = {} + self.rpm_to_modules = {} + + # Configuration + self.build_dir = build_dir + self.output_dir = output_dir + self.outprefix = outprefix + self.rhel = rhel + + # Module list + self.modlist = set() + + # Variant tracking for multi-variant merging + self.variants = [] + self.variant = variant + self.variant_idx = 0 + + # ------------------------------------------------------------------------- + # Variant Management + # ------------------------------------------------------------------------- + + def load_existing(self, filename): + """ + Load existing merged data from previous variant runs + """ + if not filename: + return + + full_path = os.path.join(self.output_dir, filename) if self.output_dir else filename + + if not os.path.exists(full_path): + return + + with open(full_path, 'rt', encoding='utf-8') as f: + data = json.load(f) + + self.variants = data.get('variants', []) + + module_map = data.get('module-map', {}) + self.module_rpm = module_map.get('module-rpm', {}) + self.rpm_to_modules = module_map.get('rpm-modules', {}) + + source_map = data.get('source-map', {}) + self.obj_to_src = source_map.get('obj-src', {}) + self.src_to_obj = source_map.get('src-obj', {}) + + def add_variant(self): + """ + Register current variant and store its index + """ + if self.variant in self.variants: + self.variant_idx = self.variants.index(self.variant) + else: + self.variant_idx = len(self.variants) + self.variants.append(self.variant) + + # ------------------------------------------------------------------------- + # Source Merging Helpers + # ------------------------------------------------------------------------- + + def _merge_sources(self, target, source): + """ + Merge source dict into target dict by combining variant index lists + """ + for src, indices in source.items(): + if src not in target: + target[src] = [] + for idx in indices: + if idx not in target[src]: + target[src].append(idx) + + def _add_source(self, sources, source_path): + """ + Add a source file to the sources dict with current variant index + + Returns True if source was added, False if filtered out + """ + source_path = os.path.normpath(source_path) + source_path = strip_path_prefixes(source_path, self.build_dir) + + if is_external_source(source_path): + return False + + sources[source_path] = [self.variant_idx] + return True + + # ------------------------------------------------------------------------- + # .cmd File Parsing + # ------------------------------------------------------------------------- + + def _parse_ko_cmd(self, ko_name): + """ + Parse .ko.cmd file to find the primary object file linked into the module + """ + ko_script = name_to_cmd(ko_name, '.ko.cmd') + lines = load_file(ko_script, build_dir=self.build_dir) + + for line in lines: + result = ko_matcher.match(line) + if result: + input_files = result.group('input_files').split() + return input_files[0] + + return None + + def _parse_o_cmd(self, o_name, translate=True): + """ + Parse .o.cmd file to extract source files that contribute to the object + + Handles: + - C/assembly compilation (gcc, clang) + - Linking (ld) - recursively parses linked objects + - Rust compilation (rustc) + - Rust source file mappings + + Args: + o_name: Object file name or .cmd file path + translate: If True, convert object name to .cmd path (e.g., kernel/fork.o + becomes kernel/.fork.o.cmd). If False, use o_name as-is. + This parameter exists to handle different kbuild behaviors + across RHEL versions - some code paths may already have .cmd + paths while others have object names that need translation. + """ + sources = {} + + o_script = name_to_cmd(o_name, '.o.cmd') if translate else o_name + lines = load_file(o_script, build_dir=self.build_dir) + + for line in lines: + result = o_matcher.match(line) + if not result: + continue + + command = result.group('command') + + if command in COMPILERS: + self._handle_compiler(result, sources) + break + + elif command == LINKER: + self._handle_linker(result, sources) + break + + elif command == RUST_COMPILER: + self._handle_rustc(result, sources) + break + + else: + self._handle_rust_source_mapping(result, sources) + break + + return sources + + def _handle_compiler(self, match_result, sources): + """ + Handle gcc/clang compilation - extract the input source file + """ + obj_input = match_result.group('input_files') + if obj_input: + self._add_source(sources, obj_input) + + def _handle_linker(self, match_result, sources): + """ + Handle ld linking - recursively parse linked objects + """ + obj_input = match_result.group('input_files') + if not obj_input: + return + + if is_mod_file_reference(obj_input): + self._merge_sources(sources, self._parse_mod_cmd(obj_input)) + else: + for obj_file in obj_input.split(): + self._merge_sources(sources, self._parse_o_cmd(obj_file)) + + def _handle_rustc(self, match_result, sources): + """ + Handle rustc compilation - extract the Rust source file + """ + rust_input = match_result.group('rust_input_file') + if rust_input: + self._add_source(sources, rust_input) + + def _handle_rust_source_mapping(self, match_result, sources): + """ + Handle Rust source file mapping (source_*.o := *.rs) + """ + source_file = match_result.group('source_file') + if source_file and source_file.endswith('.rs'): + self._add_source(sources, source_file) + + def _parse_mod_cmd(self, mod_name): + """ + Parse .mod file used by RHEL 9+ for linking modules + + These files contain a list of object files to link into the module. + """ + sources = {} + mod_name = mod_name.strip('@') + lines = load_file(mod_name, build_dir=self.build_dir) + + for line in lines: + self._merge_sources(sources, self._parse_o_cmd(line)) + + return sources + + def _parse_a_cmd(self, a_name): + """ + Parse archive .cmd file (RHEL 7 built-in.o.cmd files) + """ + sources = {} + lines = load_file(a_name, build_dir=self.build_dir) + + for line in lines: + result = o_matcher.match(line) + if not result: + continue + + ar_input = result.group('input_files') + for filename in ar_input.rsplit(): + if filename.endswith('.o'): + self._merge_sources(sources, self._parse_o_cmd(filename)) + elif filename.endswith('.a'): + self._merge_sources(sources, self._parse_a_cmd(filename)) + + return sources + + # ------------------------------------------------------------------------- + # Module Collection + # ------------------------------------------------------------------------- + + def _load_module_list(self, modorder='modules.order'): + """ + Load modules.order file and normalize paths + + RHEL 7, 8: modules.order has 'kernel/' prefix that needs to be stripped + RHEL 9, upstream: modules.order paths are used as-is + """ + lines = load_file(modorder, build_dir=self.build_dir) + + if not lines: + print(f'Warning: modules.order is empty or not found') + self.modlist = set() + return + + if self.rhel in ['9', 'upstream']: + self.modlist = set(lines) + else: + # RHEL 7, 8: strip fake 'kernel/' prefix added by kbuild + self.modlist = { + path[len(KERNEL_PREFIX):] if path.startswith(KERNEL_PREFIX) else path + for path in lines + } + + def _parse_module_objects(self): + """ + Parse all module .cmd files and extract source mappings. + + Note on RHEL version differences: + - RHEL 7/8/9: modules.order contains .ko paths (e.g., fs/exfat/exfat.ko) + - upstream: modules.order contains .o paths (e.g., fs/exfat/exfat.o) + + The normalization below ensures consistent .ko keys in output regardless + of the modules.order format. + """ + for kobj in self.modlist: + # Normalize to .ko - upstream has .o paths in modules.order + if self.rhel == 'upstream' and kobj.endswith('.o'): + module_name = kobj[:-2] + '.ko' + else: + module_name = kobj + + obj_name = self._parse_ko_cmd(module_name) + if obj_name is None: + print(f'Warning: Could not parse .ko.cmd for "{module_name}", skipping.') + continue + + obj_source = self._parse_o_cmd(obj_name) + if module_name not in self.obj_to_src: + self.obj_to_src[module_name] = {} + self._merge_sources(self.obj_to_src[module_name], obj_source) + + def collect_modules(self, no_modules=False, modorder='modules.order'): + """ + Collect source files for kernel modules + """ + if no_modules: + return + + self._load_module_list(modorder) + self._parse_module_objects() + + # ------------------------------------------------------------------------- + # Builtin Collection (RHEL version-specific) + # ------------------------------------------------------------------------- + + def _collect_builtin_rhel7(self): + """ + Collect built-in objects for RHEL 7 using *built-in.o.cmd files + """ + pattern = '*built-in.o.cmd' + cmd_list = { + strip_path_prefixes(path, self.build_dir) + for path in pathlib.Path(self.build_dir).rglob(pattern) + } + + for entry in cmd_list: + sources = self._parse_a_cmd(entry) + self._merge_sources(self.obj_to_src[VMLINUX], sources) + + def _collect_builtin_archive(self): + """ + Collect built-in objects for RHEL 8/9 using *built-in.a archives + """ + pattern = '*built-in.a' + archive_list = { + strip_path_prefixes(path, self.build_dir) + for path in pathlib.Path(self.build_dir).rglob(pattern) + } + + for entry in archive_list: + path = os.path.join(self.build_dir, entry) + objects = subprocess.check_output(['ar', 't', path]).decode().split() + + for obj in objects: + obj_path = strip_path_prefixes(obj, build_dir=self.build_dir) + sources = self._parse_o_cmd(obj_path) + self._merge_sources(self.obj_to_src[VMLINUX], sources) + + def _collect_builtin_vmlinux_archive(self): + """ + Collect built-in objects for upstream kernels using single vmlinux.a archive + """ + path = os.path.join(self.build_dir, 'vmlinux.a') + objects = subprocess.check_output(['ar', 't', path]).decode().split() + + for obj in objects: + obj_path = strip_path_prefixes(obj, build_dir=self.build_dir) + sources = self._parse_o_cmd(obj_path) + self._merge_sources(self.obj_to_src[VMLINUX], sources) + + def collect_builtin(self, no_builtin=False): + """ + Collect source files for vmlinux built-in objects + + Uses RHEL version-specific collection method via dispatch table. + """ + if VMLINUX not in self.obj_to_src: + self.obj_to_src[VMLINUX] = {} + + if no_builtin: + return + + collector_name = self.BUILTIN_COLLECTORS.get(self.rhel, '_collect_builtin_archive') + collector = getattr(self, collector_name) + collector() + + # ------------------------------------------------------------------------- + # Module-to-RPM Mapping + # ------------------------------------------------------------------------- + + def _parse_module_list_file(self, filename, rpm_name): + """ + Parse a module list file and add entries to module_rpm mapping + + Handles various formats: + - Absolute paths: /lib/modules/5.14.0/kernel/drivers/net/e1000.ko.xz + - Relative paths: kernel/drivers/net/e1000.ko + - Skips RPM spec directives (%dir, %defattr, etc.) + + Args: + filename: Path to the module list file + rpm_name: Literal RPM package name (e.g., "kernel-modules-5.14.0.rpm") + """ + if not filename or not os.path.exists(filename): + return + + with open(filename, 'rt', encoding='utf-8') as f: + for line in f: + line = line.strip() + + # Skip empty lines, comments, and RPM spec directives + if not line or line.startswith('#') or line.startswith('%'): + continue + + # Skip non-module files + if '.ko' not in line: + continue + + # Extract module name from path + module = os.path.basename(line) + # Remove .ko and any compression suffix + module = re.sub(r'\.ko(\.[^.]+)?$', '', module) + + if not module: + continue + + if module not in self.module_rpm: + self.module_rpm[module] = {} + + if rpm_name not in self.module_rpm[module]: + self.module_rpm[module][rpm_name] = [] + if self.variant_idx not in self.module_rpm[module][rpm_name]: + self.module_rpm[module][rpm_name].append(self.variant_idx) + + def collect_module_rpm(self, module_lists, vmlinux_rpm=''): + """ + Collect module-to-RPM name mappings from module list files + + Args: + module_lists: List of "rpm_name:path" strings + vmlinux_rpm: RPM package name for vmlinux + """ + if not module_lists and not vmlinux_rpm: + return + + # Add vmlinux to specified RPM + if vmlinux_rpm: + if VMLINUX not in self.module_rpm: + self.module_rpm[VMLINUX] = {} + if vmlinux_rpm not in self.module_rpm[VMLINUX]: + self.module_rpm[VMLINUX][vmlinux_rpm] = [] + if self.variant_idx not in self.module_rpm[VMLINUX][vmlinux_rpm]: + self.module_rpm[VMLINUX][vmlinux_rpm].append(self.variant_idx) + + # Process module list files + for entry in module_lists: + if ':' not in entry: + print(f'Warning: Invalid --module-list format "{entry}", expected RPM:PATH') + continue + + rpm_name, path = entry.split(':', 1) + self._parse_module_list_file(path, rpm_name) + + # ------------------------------------------------------------------------- + # Output + # ------------------------------------------------------------------------- + + def _build_reverse_map(self): + """ + Create reverse mapping (source -> {obj: [indices]}) from obj_to_src + """ + for obj, sources in self.obj_to_src.items(): + for source, indices in sources.items(): + if source not in self.src_to_obj: + self.src_to_obj[source] = {} + if obj not in self.src_to_obj[source]: + self.src_to_obj[source][obj] = [] + for idx in indices: + if idx not in self.src_to_obj[source][obj]: + self.src_to_obj[source][obj].append(idx) + + def _build_rpm_reverse_map(self): + """ + Create reverse mapping (rpm -> {module: [indices]}) from module_rpm + """ + for module, rpms in self.module_rpm.items(): + for rpm, indices in rpms.items(): + if rpm not in self.rpm_to_modules: + self.rpm_to_modules[rpm] = {} + if module not in self.rpm_to_modules[rpm]: + self.rpm_to_modules[rpm][module] = [] + for idx in indices: + if idx not in self.rpm_to_modules[rpm][module]: + self.rpm_to_modules[rpm][module].append(idx) + + def save_data(self): + """ + Save collected data to JSON file + + Output structure: + - variants: list of variant names + - source-map: source file mappings + - obj-src: object/module -> {source: [variant_indices]} + - src-obj: source -> {object/module: [variant_indices]} + - module-map: module to RPM mappings + - module-rpm: module name -> {rpm-name: [variant_indices]} + - rpm-modules: rpm-name -> {module name: [variant_indices]} + + Variant indices correspond to positions in the 'variants' list. + """ + output = { + 'variants': self.variants, + 'source-map': { + 'obj-src': self.obj_to_src, + 'src-obj': self.src_to_obj + }, + 'module-map': { + 'module-rpm': self.module_rpm, + 'rpm-modules': self.rpm_to_modules + } + } + + filename = self.outprefix + 'kernel-map.json' + with open(os.path.join(self.output_dir, filename), 'wt', encoding='utf-8') as f: + json.dump(output, f, indent=2, sort_keys=True) + + # ------------------------------------------------------------------------- + # Main Entry Point + # ------------------------------------------------------------------------- + + def collect_data(self, no_modules=False, no_builtin=False, modorder='modules.order', + input_file='', module_lists=None, vmlinux_rpm=''): + """ + Main entry point - collect all data and save to JSON + """ + self.load_existing(input_file) + self.add_variant() + self.collect_modules(no_modules, modorder) + self.collect_builtin(no_builtin) + self.collect_module_rpm(module_lists or [], vmlinux_rpm) + self._build_reverse_map() + self._build_rpm_reverse_map() + self.save_data() + + +# ============================================================================= +# Main +# ============================================================================= + +def main(): + """ + Main entry point + """ + args = parse_args() + + start = time.monotonic() + + source_map = SourceMap( + build_dir=args.directory, + output_dir=args.outputdir, + outprefix=args.prefix, + rhel=args.rhel, + variant=args.variant) + + source_map.collect_data( + no_modules=args.no_modules, + no_builtin=args.no_builtin, + input_file=args.input, + module_lists=args.module_list, + vmlinux_rpm=args.vmlinux_rpm) + + if args.time: + elapsed = time.monotonic() - start + minutes, seconds = divmod(int(elapsed), 60) + print(f'Elapsed time: {minutes:02d}:{seconds:02d}') + + +if __name__ == '__main__': + main() diff --git a/sources b/sources index 064bf1bfc..b0b22c1ee 100644 --- a/sources +++ b/sources @@ -1,3 +1,3 @@ -SHA512 (linux-7.2-rc2-22-g0e35b9b6ec0f.tar.xz) = ba3d405ba0f079911eb5c2a24c959a8ddcbff416df5e4b6904998d2b552803bfd6b7ad07c75bdc99b88ec71d4224dab536a26bee0a5ff1d593a205c9924cdb0b -SHA512 (kernel-abi-stablelists-7.2.0.tar.xz) = 991d626c74071e393ec6f9da6fedbbebf4676af560630da5d211ec160247059bf3dc73b9a4c495adfdcbacb5f08cee1cf62df4148879b59009823dd3f55b6d83 -SHA512 (kernel-kabi-dw-7.2.0.tar.xz) = 34c029d5093738ee6c9d39707b662ed676c114592bae42fc465b0205597c54d21ce5734e51d26e985cc0ddb3d8e9be8e1007888c436e86f52ec6309da7288a9c +SHA512 (linux-7.2-rc2-264-gdd3210c47e8d.tar.xz) = 577e1b2dabb0520a6410db3b6003455c0a81cbe5d6fe326839aac2696ee7050cb5e79028e43f3e3803418bf8635c8a73ef1a40957efe1f15230a510bf3059258 +SHA512 (kernel-abi-stablelists-7.2.0.tar.xz) = dc40daf2e3a7b3718343473ab3bf0eef5b53c81bc36da3bcbbc874d64038d190c1bdbfb20eb6dab940175dd970f128dd36213808d4052b7725d50b75983c5fd4 +SHA512 (kernel-kabi-dw-7.2.0.tar.xz) = 0fcab77002c598a26d377d8e8357d3747e004635bcf5c8b310bb426ed3e137cd8957c477b6a2e83b0bfe0dfc91f187a687b4790495322d6d4042961f5ff6dee2