diff --git a/python-torch.spec b/python-torch.spec
index 7bcc4bf..d3b1156 100644
--- a/python-torch.spec
+++ b/python-torch.spec
@@ -83,10 +83,14 @@
 %endif
 
 %if 0%{?fedora} > 40
+%ifarch x86_64
 %bcond_without fbgemm
 %else
 %bcond_with fbgemm
 %endif
+%else
+%bcond_with fbgemm
+%endif
 
 # For testing cuda
 %ifarch x86_64