diff --git a/optimum/gptq/quantizer.py b/optimum/gptq/quantizer.py
index 504153ea8b..db77ac0341 100644
--- a/optimum/gptq/quantizer.py
+++ b/optimum/gptq/quantizer.py
@@ -665,7 +665,7 @@ def tmp(_, input, output):
                 del layer_inputs
                 layer_inputs = []
             torch.cuda.empty_cache()
-            if hasattr(torch, "xpu"):
+            if hasattr(torch, "xpu") and torch.xpu.is_available():
                 torch.xpu.empty_cache()
 
         if self.bits == 4: