diff --git a/.gitignore b/.gitignore index 5fda907..1ef9c4f 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,10 @@ /kineto-5e75018.tar.gz /pytorch-v2.8.0.tar.gz /v1.18.0.tar.gz +/pytorch-715dca6.tar.gz +/pytorch-fd36458.tar.gz +/pytorch-0fabc3b.tar.gz +/pytorch-v2.9.0.tar.gz +/pytorch-v2.9.1.tar.gz +/v3.19.6.tar.gz +/v80.9.0.tar.gz diff --git a/pyproject.toml b/pyproject.toml index ccf9c2a..925742b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,49 +1,105 @@ +# Package ###################################################################### + +[build-system] +requires = [ + # 70.1.0: min version for integrated bdist_wheel command from wheel package + # 77.0.0: min version for SPDX expression support for project.license + "setuptools>=70.1.0,<80.0", + "cmake>=3.27", + "ninja", + "numpy", + "packaging", + "pyyaml", + "requests", + "six", # dependency chain: NNPACK -> PeachPy -> six + "typing-extensions>=4.10.0", +] +build-backend = "setuptools.build_meta" + +[dependency-groups] +dev = [ + # This list should be kept in sync with the requirements-build.txt + # in PyTorch root until the project fully migrates to pyproject.toml + # after which this can be removed as it is already specified in the + # [build-system] section + "setuptools>=70.1.0,<80.0", # setuptools develop deprecated on 80.0 + "cmake>=3.27", + "ninja", + "numpy", + "packaging", + "pyyaml", + "requests", + "six", # dependency chain: NNPACK -> PeachPy -> six + "typing-extensions>=4.10.0", + + # This list should be kept in sync with the requirements.txt in + # PyTorch root until the project fully migrates to pyproject.toml + "build[uv]", + "expecttest>=0.3.0", + "filelock", + "fsspec>=0.8.5", + "hypothesis", + "jinja2", + "lintrunner; platform_machine != 's390x' and platform_machine != 'riscv64'", + "networkx>=2.5.1", + "optree>=0.13.0", + "psutil", + "sympy>=1.13.3", + "typing-extensions>=4.13.2", + "wheel", +] + [project] name = "torch" -requires-python = ">=3.9" -license = {text = "BSD-3-Clause"} +description = "Tensors and Dynamic neural networks in Python with strong GPU acceleration" +readme = "README.md" +requires-python = ">=3.10" +# TODO: change to `license = "BSD-3-Clause"` and enable PEP 639 after pinning setuptools>=77 +# FIXME: As of 2025.06.20, it is hard to ensure the minimum version of setuptools in our CI environment. +# TOML-table-based license deprecated in setuptools>=77, and the deprecation warning will be changed +# to an error on 2026.02.18. See also: https://github.com/pypa/setuptools/issues/4903 +license = { text = "BSD-3-Clause" } +authors = [{ name = "PyTorch Team", email = "packages@pytorch.org" }] +keywords = ["pytorch", "machine learning"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Mathematics", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Programming Language :: C++", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] dynamic = [ - "authors", - "classifiers", "entry-points", "dependencies", - "description", - "keywords", - "optional-dependencies", - "readme", "scripts", "version", ] [project.urls] -Homepage = "https://pytorch.org/" -Documentation = "https://pytorch.org/docs/" -Source = "https://github.com/pytorch/pytorch" -Forum = "https://discuss.pytorch.org/" +Homepage = "https://pytorch.org" +Repository = "https://github.com/pytorch/pytorch" +Documentation = "https://pytorch.org/docs" +"Issue Tracker" = "https://github.com/pytorch/pytorch/issues" +Forum = "https://discuss.pytorch.org" +[project.optional-dependencies] +optree = ["optree>=0.13.0"] +opt-einsum = ["opt-einsum>=3.3"] +pyyaml = ["pyyaml"] -[build-system] -requires = [ - # After 75.8.2 dropped dep disttools API. Please fix - # API temporarily restored and shim used. Please fix - # Setuptools will drop support for setup.py past 80 - # min version for recursive glob package data support - "setuptools>=62.3.0,<80.0", - "wheel", - "astunparse", - "numpy", - "ninja", - "pyyaml", - "cmake", - "typing-extensions>=4.10.0", - "requests", -] -# Use legacy backend to import local packages in setup.py -build-backend = "setuptools.build_meta:__legacy__" - - -[tool.black] -line-length = 88 +# Linter tools ################################################################# [tool.isort] src_paths = ["caffe2", "torch", "torchgen", "functorch", "test"] @@ -59,12 +115,10 @@ multi_line_output = 3 include_trailing_comma = true combine_as_imports = true - [tool.usort.known] first_party = ["caffe2", "torch", "torchgen", "functorch", "test"] standard_library = ["typing_extensions"] - [tool.ruff] line-length = 88 src = ["caffe2", "torch", "torchgen", "functorch", "test"] @@ -105,6 +159,7 @@ ignore = [ "E741", "EXE001", "F405", + "FURB122", # writelines # these ignores are from flake8-logging-format; please fix! "G101", # these ignores are from ruff NPY; please fix! @@ -127,7 +182,15 @@ ignore = [ "SIM117", "SIM118", "UP007", # keep-runtime-typing + "UP045", # keep-runtime-typing "TC006", + # TODO: Remove Python-3.10 specific suppressions + "B905", + "UP035", + "UP036", + "UP038", + "UP041", + "FURB161", ] select = [ "B", @@ -208,6 +271,10 @@ select = [ "YTT", ] +[tool.ruff.lint.pyupgrade] +# Preserve types, even if a file imports `from __future__ import annotations`. +keep-runtime-typing = true + [tool.ruff.lint.per-file-ignores] "__init__.py" = [ "F401", diff --git a/python-torch.spec b/python-torch.spec index de807e5..09d6699 100644 --- a/python-torch.spec +++ b/python-torch.spec @@ -6,16 +6,17 @@ # So pre releases can be tried %bcond_with gitcommit %if %{with gitcommit} -# v2.8.0-rc8 -%global commit0 a1cb3cc05d46d198467bebbb6e8fba50a325d4e7 +# v2.9.0-rc9 +%global commit0 0fabc3ba44823f257e70ce397d989c8de5e362c1 %global shortcommit0 %(c=%{commit0}; echo ${c:0:7}) -%global date0 20250723 -%global pypi_version 2.8.0 +%global date0 20251008 +%global pypi_version 2.9.0 %global flatbuffers_version 24.12.23 %global miniz_version 3.0.2 %global pybind11_version 2.13.6 +%global rc_tag -rc9 %else -%global pypi_version 2.8.0 +%global pypi_version 2.9.1 %global flatbuffers_version 24.12.23 %global miniz_version 3.0.2 %global pybind11_version 2.13.6 @@ -33,7 +34,6 @@ %endif # For testing distributed+rccl etc. -%bcond_without rccl %bcond_with gloo %bcond_without mpi %bcond_without tensorpipe @@ -48,10 +48,17 @@ %bcond_with httplib %bcond_with kineto +# In fedora, not in rhel/epel or requires a newer version %if 0%{?fedora} +%bcond_without eigen3 %bcond_without onnx +%bcond_without protobuf +%bcond_without setuptools %else +%bcond_with eigen3 %bcond_with onnx +%bcond_with protobuf +%bcond_with setuptools %endif Name: python-%{pypi_name} @@ -60,7 +67,7 @@ Version: %{pypi_version}^git%{date0}.%{shortcommit0} %else Version: %{pypi_version} %endif -Release: %autorelease +Release: 1%{?dist} Summary: PyTorch AI/ML framework # See license.txt for license details License: BSD-3-Clause AND BSD-2-Clause AND 0BSD AND Apache-2.0 AND MIT AND BSL-1.0 AND GPL-3.0-or-later AND Zlib @@ -103,30 +110,33 @@ Source70: https://github.com/yhirose/cpp-httplib/archive/%{hl_commit}/cpp- Source80: https://github.com/pytorch/kineto/archive/%{ki_commit}/kineto-%{ki_scommit}.tar.gz %endif -%if %{without onnx} %global ox_ver 1.18.0 Source90: https://github.com/onnx/onnx/archive/refs/tags/v%{ox_ver}.tar.gz -%endif -# https://github.com/pytorch/pytorch/issues/150187 -Patch11: 0001-Add-cmake-variable-USE_ROCM_CK.patch -# https://github.com/pytorch/pytorch/issues/156595 -# Patch12: 0001-Use-horrible-dynamo-stub.patch -Patch12: 0001-Fix-compilation-and-import-torch-issues-for-cpython-.patch +%global pb_ver 3.19.6 +Source100: https://github.com/protocolbuffers/protobuf/archive/refs/tags/v%{pb_ver}.tar.gz + +# A problem on RHEL 10 +# + /usr/bin/python3 setup.py build '--executable=/usr/bin/python3 -sP' +# Traceback (most recent call last): +# File "/root/rpmbuild/BUILD/pytorch-v2.9.1/setup.py", line 288, in +# import setuptools.command.bdist_wheel +# ModuleNotFoundError: No module named 'setuptools.command.bdist_wheel' +# This support came in 70.1.0, RHEL 10.2 has 69.0.3 +# Use ELN version +%global st_ver 80.9.0 +Source110: https://github.com/pypa/setuptools/archive/refs/tags/v%{st_ver}.tar.gz -# 2.8 has this problem on aarch64 -# /builddir/build/BUILD/pytorch-v2.8.0/c10/util/FunctionRef.h:45:3: error: unrecognizable insn: -# (insn 3196 3195 3197 78 (set (reg:VNx8BF 1368 [ _494 ]) -# (unspec:VNx8BF [ -# (reg:VNx8BF 1374) -# (reg:VNx8BF 1375) -# ] UNSPEC_IORF)) "/builddir/build/BUILD/pytorch-v2.8.0/aten/src/ATen/cpu/vec/sve/vec_bfloat16.h":228:31 discrim 1 -1 -ExclusiveArch: x86_64 %global toolchain gcc %global _lto_cflags %nil +ExclusiveArch: x86_64 + BuildRequires: cmake +BuildRequires: cpuinfo-devel +%if %{with eigen3} BuildRequires: eigen3-devel +%endif BuildRequires: flexiblas-devel BuildRequires: fmt-devel BuildRequires: foxi-devel @@ -139,21 +149,22 @@ BuildRequires: gloo-devel BuildRequires: json-devel BuildRequires: libomp-devel +BuildRequires: moodycamel-concurrentqueue-devel BuildRequires: numactl-devel -BuildRequires: ninja-build %if %{with onnx} BuildRequires: onnx-devel %endif %if %{with mpi} BuildRequires: openmpi-devel %endif +%if %{with protobuf} BuildRequires: protobuf-devel +%endif BuildRequires: sleef-devel BuildRequires: valgrind-devel BuildRequires: pocketfft-devel BuildRequires: pthreadpool-devel -BuildRequires: cpuinfo-devel BuildRequires: FP16-devel BuildRequires: fxdiv-devel BuildRequires: psimd-devel @@ -166,14 +177,19 @@ BuildRequires: python3dist(networkx) BuildRequires: python3dist(numpy) BuildRequires: python3dist(pip) BuildRequires: python3dist(pyyaml) +%if %{with setuptools} BuildRequires: python3dist(setuptools) -BuildRequires: python3dist(sphinx) +%endif BuildRequires: python3dist(typing-extensions) +# Packages missing for RHEL/EPEL %if 0%{?fedora} BuildRequires: python3-pybind11 BuildRequires: python3dist(fsspec) +BuildRequires: python3dist(sphinx) BuildRequires: python3dist(sympy) +# New for 2.9 / EPEL 10.2 +BuildRequires: ninja-build %endif %if %{with rocm} @@ -190,9 +206,7 @@ BuildRequires: miopen-devel BuildRequires: rocblas-devel BuildRequires: rocrand-devel BuildRequires: rocfft-devel -%if %{with rccl} BuildRequires: rccl-devel -%endif BuildRequires: rocprim-devel BuildRequires: rocm-cmake BuildRequires: rocm-comgr-devel @@ -277,6 +291,18 @@ cp %{SOURCE1000} . %else %autosetup -p1 -n pytorch-v%{version} + +# GitHub release tarballs identify the version as an alpha, so replace that +echo "%{pypi_version}" > version.txt + +%endif + +%if %{without setuptools} +mkdir -p extra-python +cd extra-python +tar xf %{SOURCE110} +mv setuptools-* setuptools +cd .. %endif # Remove bundled egg-info @@ -330,8 +356,14 @@ rm -rf third_party/onnx/* cp -r onnx-*/* third_party/onnx/ %endif +%if %{without protobuf} +tar xf %{SOURCE100} +rm -rf third_party/protobuf/* +cp -r protobuf-*/* third_party/protobuf/ +%endif + # Adjust for the hipblaslt's we build -sed -i -e 's@"gfx90a", "gfx940", "gfx941", "gfx942"@"gfx90a", "gfx1103", "gfx1150", "gfx1151", "gfx1100", "gfx1101", "gfx1200", "gfx1201"@' aten/src/ATen/native/cuda/Blas.cpp +sed -i -e 's@"gfx1100", "gfx1101", "gfx1200", "gfx1201", "gfx908",@"gfx1100", "gfx1101", "gfx1200", "gfx1201", "gfx1151",@' aten/src/ATen/native/cuda/Blas.cpp %if 0%{?rhel} # In RHEL but too old @@ -423,6 +455,10 @@ mv third_party/kineto . mv third_party/onnx . %endif +%if %{without protobuf} +mv third_party/protobuf . +%endif + %if %{with test} mv third_party/googletest . %endif @@ -455,6 +491,10 @@ mv kineto third_party mv onnx third_party %endif +%if %{without protobuf} +mv protobuf third_party +%endif + %if %{with test} mv googletest third_party %endif @@ -500,16 +540,27 @@ sed -i -e 's@HIP 1.0@HIP MODULE@' cmake/public/LoadHIP.cmake %endif +# moodycamel include path needs adjusting to use the system's +sed -i -e 's@${PROJECT_SOURCE_DIR}/third_party/concurrentqueue@/usr/include/concurrentqueue@' cmake/Dependencies.cmake + %build +# Export the arches +# echo "%%pytorch_arches %pt_arches" > macros.pytorch + # # Control the number of jobs # # The build can fail if too many threads exceed the physical memory -# So count core and and memory and increase the build memory util the build succeeds +# Run at least one thread, more if CPU & memory resources are available. # +%ifarch x86_64 # Real cores, No hyperthreading COMPILE_JOBS=`cat /proc/cpuinfo | grep -m 1 'cpu cores' | awk '{ print $4 }'` +%else +# cpuinfo format varies on other arches, fall back to nproc +COMPILE_JOBS=`nproc` +%endif if [ ${COMPILE_JOBS}x = x ]; then COMPILE_JOBS=1 fi @@ -536,7 +587,10 @@ export MAX_JOBS=$COMPILE_JOBS # Manually set this hardening flag export CMAKE_EXE_LINKER_FLAGS=-pie +%if %{with protobuf} export BUILD_CUSTOM_PROTOBUF=OFF +%endif + export BUILD_NVFUSER=OFF export BUILD_SHARED_LIBS=ON export BUILD_TEST=OFF @@ -566,7 +620,9 @@ export USE_OPENMP=ON export USE_PYTORCH_QNNPACK=OFF export USE_ROCM=OFF export USE_SYSTEM_SLEEF=ON +%if %{with eigen3} export USE_SYSTEM_EIGEN_INSTALL=ON +%endif %if %{with onnx} export USE_SYSTEM_ONNX=ON %endif @@ -601,24 +657,16 @@ export USE_MPI=ON export BUILD_TEST=ON %endif -# Why we are using py3_ vs pyproject_ -# -# current pyproject problem with mock -# + /usr/bin/python3 -Bs /usr/lib/rpm/redhat/pyproject_wheel.py /builddir/build/BUILD/pytorch-v2.1.0/pyproject-wheeldir -# /usr/bin/python3: No module named pip -# Adding pip to build requires does not fix -# -# See BZ 2244862 - %if %{with rocm} export USE_ROCM=ON -export USE_ROCM_CK=OFF +export USE_ROCM_CK_SDPA=OFF +export USE_ROCM_CK_GEMM=OFF +export USE_FBGEMM_GENAI=OFF + export USE_MAGMA=ON export HIP_PATH=`hipconfig -p` export ROCM_PATH=`hipconfig -R` -RESOURCE_DIR=`%{rocmllvm_bindir}/clang -print-resource-dir` -export DEVICE_LIB_PATH=${RESOURCE_DIR}/amdgcn/bitcode # pytorch uses clang, not hipcc export HIP_CLANG_PATH=%{rocmllvm_bindir} @@ -626,6 +674,15 @@ export PYTORCH_ROCM_ARCH=%{rocm_gpu_list_default} %endif +%if %{without setuptools} +cd extra-python +EXTRA_PYTHON=$PWD +cd setuptools +python3 setup.py install --root $EXTRA_PYTHON +cd ../.. +export PYTHONPATH=${EXTRA_PYTHON}%{python3_sitelib}:$PYTHONPATH +%endif + %if 0%{?fedora} %pyproject_wheel %else @@ -640,34 +697,40 @@ export USE_ROCM=ON export USE_ROCM_CK=OFF export HIP_PATH=`hipconfig -p` export ROCM_PATH=`hipconfig -R` -RESOURCE_DIR=`%{rocmllvm_bindir}/clang -print-resource-dir` -export DEVICE_LIB_PATH=${RESOURCE_DIR}/amdgcn/bitcode # pytorch uses clang, not hipcc export HIP_CLANG_PATH=%{rocmllvm_bindir} export PYTORCH_ROCM_ARCH=%{rocm_gpu_list_default} - %endif %if 0%{?fedora} %pyproject_install %pyproject_save_files '*torch*' %else -%py3_install -%endif +# In 2.9+ the 'setup.py install' command is gone +# + /usr/bin/python3 setup.py install -O1 --skip-build --root /builddir/build/BUILDROOT/python-torch-2.9.1-1.el10.x86_64 --prefix /usr +# WARNING: Redirecting 'python setup.py install' to 'pip install . -v --no-build-isolation', for more inf +# So we can no longer use py3_install +# The replacement from pytorch does not map arguements +# cobble together a new install based on pyproject_install +PYTHONPATH=$PWD/extra-python/%{python3_sitelib}:$PYTHONPATH \ + TMPDIR="$PWD/build" \ + %{__python3} -m pip install --root %{buildroot} --prefix %{_prefix} --no-deps --disable-pip-version-check --progress-bar off --verbose --ignore-installed --no-warn-script-location --no-index --no-cache-dir --no-build-isolation . + +%endif %check # Not working yet # pyproject_check_import torch -# Do not remote the empty files - %files -n python3-%{pypi_name} %license LICENSE %doc README.md %{_bindir}/torchrun +%if 0%{?rhel} %{_bindir}/torchfrtrace +%endif %{python3_sitearch}/%{pypi_name}* %{python3_sitearch}/functorch diff --git a/sources b/sources index 5c15ab4..e184e73 100644 --- a/sources +++ b/sources @@ -12,3 +12,10 @@ SHA512 (v24.12.23.tar.gz) = f97762ba41b9cfef648e93932fd789324c6bb6ebc5b7aeca8185 SHA512 (kineto-5e75018.tar.gz) = 921b96a56e01d69895b79e67582d8977ed6f873573ab41557c5d026ada5d1f6365e4ed0a0c6804057c52e92510749fc58619f554a164c1ba9d8cd13e789bebd0 SHA512 (pytorch-v2.8.0.tar.gz) = 791e658eab87fb957f025558cb9f925078d2426ab7b6f60771d9841dfb691f67d905ba1330a800008efe7c938b6c69bdc52232bccfe8d4860e795a532cd69d28 SHA512 (v1.18.0.tar.gz) = 2f38664947c8d1efc40620a7c1b1953d2aa4b0a37b67c4886b86e77c1d697363c26413413ddda8eabc545892fb1bcb43afc7e93e62f0901527524a2727e1ea8d +SHA512 (pytorch-715dca6.tar.gz) = 09c9aae54fab3eb17901fc3226fece1c13f41cb8e45a2cb066021823abeb8d27c340993088e01d8e55bb37ed5f94334ec31e6c539cddfacbad157abd27c5e907 +SHA512 (pytorch-fd36458.tar.gz) = acbb7475b92ad4a8e8d779f3745da22d8438e4c5ef2d6e76d71c987789f2752c8aef7022c87c9a74640fe4f9c1f1a61a3f12a796f63b1e6be24da8e5aacf37dc +SHA512 (pytorch-0fabc3b.tar.gz) = 2e87975de0bf6f3dcede168b379e1928712bca16170c2a8ee7d63459f53086c01baac05e0763e4d5d28cdaf1c7d8912225ee06adeff96ead4f6f456ee174b341 +SHA512 (pytorch-v2.9.0.tar.gz) = ae989e3a7fe30f9ea90944dc25e21ca92f2a94ee40d8de974a168c292d82c16ee8920624eff91a85755469ad05473dce0f85893e3ed7794ec5c6bdd89cbd2023 +SHA512 (pytorch-v2.9.1.tar.gz) = 88de0289fa2760abd69bef505b5ae3b6d7ff176b415cbb31bbc89ce5476a3800b322a97c4490f270f8b89657aff931bf9a5516202b268e0bb8b1f63dbb87b34a +SHA512 (v3.19.6.tar.gz) = 8f92242f2be8e1bbfba41341c87709ad91ad83b8b3e3df88bb430411541d3399295f49291fd52b50e3487b0fce33181cb4d175685fd25aac72adfaee26a612d4 +SHA512 (v80.9.0.tar.gz) = e67c6c5e74691e65ecbf24fd19c1da545434ac1674f37c917f5ecc282a00bffc62d64164a885d19f177e99a3fa53db196e2be1cf0aa6811027e61d05119682da