Fix Apertus model crash on float16 hardware

Initialize XIELU activation with correct dtype from config (using config.dtype instead of default bfloat16) to prevent promotion to float32 and subsequent crashes on Turing/float16 GPUs.

Author: wasertech

src/transformers/models/apertus/modeling_apertus.py

@@ -49,6 +49,10 @@ def __init__(self, config):
         self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
         self.act_fn = ACT2FN[config.hidden_act]
+        if config.hidden_act == "xielu":
+            from ...activations import ACT2CLS
+
+            self.act_fn = ACT2CLS["xielu"](dtype=config.dtype)
 
     def forward(self, x):
         return self.down_proj(self.act_fn(self.up_proj(x)))

src/transformers/models/apertus/modular_apertus.py

@@ -192,9 +192,13 @@ def __init__(
 
 class ApertusMLP(NemotronMLP):
     def __init__(self, config):
-        super().__init__()
+        super().__init__(config)
         self.up_proj = nn.Linear(self.hidden_size, self.intermediate_size, bias=False)
         self.down_proj = nn.Linear(self.intermediate_size, self.hidden_size, bias=False)
+        if config.hidden_act == "xielu":
+            from ...activations import ACT2CLS
+
+            self.act_fn = ACT2CLS["xielu"](dtype=config.dtype)
 
 
 class ApertusRMSNorm(LlamaRMSNorm):

tests/models/apertus/test_modeling_apertus.py

@@ -60,3 +60,6 @@ class ApertusModelTest(CausalLMModelTest, unittest.TestCase):
 @slow
 class ApertusIntegrationTest(unittest.TestCase):
     pass
+
+
+