From bd53a93037faec8d5639d353a3b192985d05f6e4 Mon Sep 17 00:00:00 2001 From: DanielSun11 <1395924413@qq.com> Date: Tue, 30 Jun 2026 00:30:03 +0800 Subject: [PATCH 1/2] Set init_method_std default to sqrt(0.3333/hidden_size) - Change init_method_std default from 0.02 to None - In __post_init__, when init_method_std or embedding_init_method_std is None, set it to math.sqrt(0.3333 / hidden_size) --- src/paddlefleet/transformer/transformer_config.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/paddlefleet/transformer/transformer_config.py b/src/paddlefleet/transformer/transformer_config.py index a06658de26..0cd5874649 100644 --- a/src/paddlefleet/transformer/transformer_config.py +++ b/src/paddlefleet/transformer/transformer_config.py @@ -631,9 +631,10 @@ class TransformerConfig(ModelParallelConfig): will be set to paddlefleet.utils.scaled_init_method_normal(init_method_std) which is paddle nn init normal with mean=0.0 and std=init_method_std / math.sqrt(2.0 * num_hidden_layers).""" - init_method_std: float = 0.02 + init_method_std: float = None """Standard deviation of the zero mean normal for the default initialization method, not used if - init_method and output_layer_init_method are provided.""" + init_method and output_layer_init_method are provided. If None, will be set to + math.sqrt(0.3333 / hidden_size).""" embedding_init_method: callable = None """ @@ -984,6 +985,9 @@ def __post_init__(self): details. """ super().__post_init__() + + if self.init_method_std is None: + self.init_method_std = math.sqrt(0.3333 / self.hidden_size) if self.enable_mtp_magic_send: assert self.num_nextn_predict_layers == 1, ( "enable_mtp_magic_send only supports num_nextn_predict_layers=1" From 29a95b07abf3762285805d8ecf8e091bbd96a141 Mon Sep 17 00:00:00 2001 From: DanielSun11 <1395924413@qq.com> Date: Tue, 30 Jun 2026 11:28:25 +0800 Subject: [PATCH 2/2] fix zero div --- src/paddlefleet/transformer/transformer_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/paddlefleet/transformer/transformer_config.py b/src/paddlefleet/transformer/transformer_config.py index 0cd5874649..0a1c3ba19c 100644 --- a/src/paddlefleet/transformer/transformer_config.py +++ b/src/paddlefleet/transformer/transformer_config.py @@ -987,7 +987,10 @@ def __post_init__(self): super().__post_init__() if self.init_method_std is None: - self.init_method_std = math.sqrt(0.3333 / self.hidden_size) + if self.hidden_size == 0: + self.init_method_std = 0.02 + else: + self.init_method_std = math.sqrt(0.3333 / self.hidden_size) if self.enable_mtp_magic_send: assert self.num_nextn_predict_layers == 1, ( "enable_mtp_magic_send only supports num_nextn_predict_layers=1"