diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_hash.txt b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_hash.txt new file mode 100644 index 0000000000..f420ea803b --- /dev/null +++ b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_hash.txt @@ -0,0 +1 @@ +852cea4ff0a26014e69c23284ac770139229c5bc0086b85fbb964909841b1351 \ No newline at end of file diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_net.json b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/input_meta.py b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/input_tensor_constraints.py b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/model.py b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/model.py new file mode 100644 index 0000000000..38a2a621a3 --- /dev/null +++ b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/model.py @@ -0,0 +1,4829 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_kwargs_input_ids_: torch.Tensor, + L_self_modules_model_modules_embed_tokens_parameters_weight_: torch.nn.parameter.Parameter, + L_kwargs_attention_mask_: torch.Tensor, + L_self_modules_model_modules_rotary_emb_buffers_inv_freq_: torch.Tensor, + L_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_model_modules_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_lm_head_parameters_weight_: torch.nn.parameter.Parameter, + ): + l_kwargs_input_ids_ = L_kwargs_input_ids_ + l_self_modules_model_modules_embed_tokens_parameters_weight_ = ( + L_self_modules_model_modules_embed_tokens_parameters_weight_ + ) + l_kwargs_attention_mask_ = L_kwargs_attention_mask_ + l_self_modules_model_modules_rotary_emb_buffers_inv_freq_ = ( + L_self_modules_model_modules_rotary_emb_buffers_inv_freq_ + ) + l_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_ + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ = L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ + l_self_modules_model_modules_norm_parameters_weight_ = ( + L_self_modules_model_modules_norm_parameters_weight_ + ) + l_self_modules_lm_head_parameters_weight_ = ( + L_self_modules_lm_head_parameters_weight_ + ) + inputs_embeds = torch.nn.functional.embedding( + l_kwargs_input_ids_, + l_self_modules_model_modules_embed_tokens_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_kwargs_input_ids_ = ( + l_self_modules_model_modules_embed_tokens_parameters_weight_ + ) = None + cache_position = torch.arange(0, 21, device=device(type="cpu")) + position_ids = cache_position.unsqueeze(0) + attention_mask = l_kwargs_attention_mask_.to( + device=device(type="cpu"), dtype=torch.bool + ) + l_kwargs_attention_mask_ = None + kv_arange = torch.arange(21, device=device(type="cpu")) + kv_arange += 0 + kv_arange_1 = kv_arange + kv_arange = None + batch_arange = torch.arange(1, device=device(type="cpu")) + head_arange = torch.arange(1, device=device(type="cpu")) + lazy_load_decompositions = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions = None + _vmap_increment_nesting = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting = None + child = torch._C._functorch._add_batch_dim(batch_arange, 0, 1) + batch_arange = None + lazy_load_decompositions_1 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_1 = None + _vmap_increment_nesting_1 = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting_1 = None + child_1 = torch._C._functorch._add_batch_dim(head_arange, 0, 2) + head_arange = child_1 = None + lazy_load_decompositions_2 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_2 = None + _vmap_increment_nesting_2 = torch._C._functorch._vmap_increment_nesting( + 21, "error" + ) + _vmap_increment_nesting_2 = None + child_2 = torch._C._functorch._add_batch_dim(cache_position, 0, 3) + cache_position = None + lazy_load_decompositions_3 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_3 = None + _vmap_increment_nesting_3 = torch._C._functorch._vmap_increment_nesting( + 21, "error" + ) + _vmap_increment_nesting_3 = None + child_3 = torch._C._functorch._add_batch_dim(kv_arange_1, 0, 4) + kv_arange_1 = None + result = child_2.new_ones((), dtype=torch.bool) + le = child_3.le(child_2) + child_2 = None + result_1 = result.__and__(le) + result = le = None + function_ctx = torch.autograd.function.FunctionCtx() + function_ctx = None + index = torch.ops.aten.index(attention_mask, [child, child_3]) + attention_mask = child = child_3 = None + result_2 = result_1.__and__(index) + result_1 = index = None + batched_outputs = torch._C._functorch._remove_batch_dim(result_2, 4, 21, 0) + result_2 = None + _vmap_decrement_nesting = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting = None + batched_outputs_1 = torch._C._functorch._remove_batch_dim( + batched_outputs, 3, 21, 0 + ) + batched_outputs = None + _vmap_decrement_nesting_1 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_1 = None + batched_outputs_2 = torch._C._functorch._remove_batch_dim( + batched_outputs_1, 2, 1, 0 + ) + batched_outputs_1 = None + _vmap_decrement_nesting_2 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_2 = None + causal_mask = torch._C._functorch._remove_batch_dim(batched_outputs_2, 1, 1, 0) + batched_outputs_2 = None + _vmap_decrement_nesting_3 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_3 = None + getitem = l_self_modules_model_modules_rotary_emb_buffers_inv_freq_[ + (None, slice(None, None, None), None) + ] + l_self_modules_model_modules_rotary_emb_buffers_inv_freq_ = None + float_1 = getitem.float() + getitem = None + expand = float_1.expand(1, -1, 1) + float_1 = None + inv_freq_expanded = expand.to(device(type="cpu")) + expand = None + getitem_1 = position_ids[ + (slice(None, None, None), None, slice(None, None, None)) + ] + position_ids = None + position_ids_expanded = getitem_1.float() + getitem_1 = None + _enter_autocast = torch.amp.autocast_mode._enter_autocast( + "cpu", None, False, None + ) + float_3 = inv_freq_expanded.float() + inv_freq_expanded = None + float_4 = position_ids_expanded.float() + position_ids_expanded = None + matmul = float_3 @ float_4 + float_3 = float_4 = None + freqs = matmul.transpose(1, 2) + matmul = None + emb = torch.cat((freqs, freqs), dim=-1) + freqs = None + cos = emb.cos() + cos_1 = cos * 1.0 + cos = None + sin = emb.sin() + emb = None + sin_1 = sin * 1.0 + sin = None + _exit_autocast = torch.amp.autocast_mode._exit_autocast(_enter_autocast) + _enter_autocast = _exit_autocast = None + cos_2 = cos_1.to(dtype=torch.float16) + cos_1 = None + sin_2 = sin_1.to(dtype=torch.float16) + sin_1 = None + _log_api_usage_once = torch._C._log_api_usage_once("python.nn_module") + _log_api_usage_once = None + hidden_states = inputs_embeds.to(torch.float32) + pow_1 = hidden_states.pow(2) + variance = pow_1.mean(-1, keepdim=True) + pow_1 = None + add = variance + 1e-05 + variance = None + rsqrt = torch.rsqrt(add) + add = None + hidden_states_1 = hidden_states * rsqrt + hidden_states = rsqrt = None + to_5 = hidden_states_1.to(torch.float16) + hidden_states_1 = None + hidden_states_2 = ( + l_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ + * to_5 + ) + l_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_ = ( + to_5 + ) = None + linear = torch._C._nn.linear( + hidden_states_2, + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view = linear.view((1, 21, -1, 64)) + linear = None + query_states = view.transpose(1, 2) + view = None + linear_1 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_1 = linear_1.view((1, 21, -1, 64)) + linear_1 = None + key_states = view_1.transpose(1, 2) + view_1 = None + linear_2 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_2 = l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_2 = linear_2.view((1, 21, -1, 64)) + linear_2 = None + value_states = view_2.transpose(1, 2) + view_2 = None + cos_3 = cos_2.unsqueeze(1) + sin_3 = sin_2.unsqueeze(1) + mul_4 = query_states * cos_3 + x1 = query_states[(Ellipsis, slice(None, 32, None))] + x2 = query_states[(Ellipsis, slice(32, None, None))] + query_states = None + neg = -x2 + x2 = None + cat_1 = torch.cat((neg, x1), dim=-1) + neg = x1 = None + mul_5 = cat_1 * sin_3 + cat_1 = None + q_embed = mul_4 + mul_5 + mul_4 = mul_5 = None + mul_6 = key_states * cos_3 + cos_3 = None + x1_1 = key_states[(Ellipsis, slice(None, 32, None))] + x2_1 = key_states[(Ellipsis, slice(32, None, None))] + key_states = None + neg_1 = -x2_1 + x2_1 = None + cat_2 = torch.cat((neg_1, x1_1), dim=-1) + neg_1 = x1_1 = None + mul_7 = cat_2 * sin_3 + cat_2 = sin_3 = None + k_embed = mul_6 + mul_7 + mul_6 = mul_7 = None + getitem_6 = k_embed[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed = None + hidden_states_3 = getitem_6.expand(1, 4, 8, 21, 64) + getitem_6 = None + key = hidden_states_3.reshape(1, 32, 21, 64) + hidden_states_3 = None + getitem_7 = value_states[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states = None + hidden_states_4 = getitem_7.expand(1, 4, 8, 21, 64) + getitem_7 = None + value = hidden_states_4.reshape(1, 32, 21, 64) + hidden_states_4 = None + attention_mask_1 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query = q_embed.contiguous() + q_embed = None + key_1 = key.contiguous() + key = None + value_1 = value.contiguous() + value = None + attn_output = torch._C._nn.scaled_dot_product_attention( + query, + key_1, + value_1, + attn_mask=attention_mask_1, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query = key_1 = value_1 = attention_mask_1 = None + transpose_4 = attn_output.transpose(1, 2) + attn_output = None + attn_output_1 = transpose_4.contiguous() + transpose_4 = None + reshape_2 = attn_output_1.reshape(1, 21, -1) + attn_output_1 = None + attn_output_2 = reshape_2.contiguous() + reshape_2 = None + attn_output_3 = torch._C._nn.linear( + attn_output_2, + l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_2 = l_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_5 = inputs_embeds + attn_output_3 + inputs_embeds = attn_output_3 = None + hidden_states_6 = hidden_states_5.to(torch.float32) + pow_2 = hidden_states_6.pow(2) + variance_1 = pow_2.mean(-1, keepdim=True) + pow_2 = None + add_4 = variance_1 + 1e-05 + variance_1 = None + rsqrt_1 = torch.rsqrt(add_4) + add_4 = None + hidden_states_7 = hidden_states_6 * rsqrt_1 + hidden_states_6 = rsqrt_1 = None + to_7 = hidden_states_7.to(torch.float16) + hidden_states_7 = None + hidden_states_8 = ( + l_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ + * to_7 + ) + l_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_ = ( + to_7 + ) = None + linear_4 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu = torch.nn.functional.silu(linear_4, inplace=False) + linear_4 = None + linear_5 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_8 = l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_10 = silu * linear_5 + silu = linear_5 = None + down_proj = torch._C._nn.linear( + mul_10, + l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_10 = l_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_9 = hidden_states_5 + down_proj + hidden_states_5 = down_proj = None + hidden_states_10 = hidden_states_9.to(torch.float32) + pow_3 = hidden_states_10.pow(2) + variance_2 = pow_3.mean(-1, keepdim=True) + pow_3 = None + add_6 = variance_2 + 1e-05 + variance_2 = None + rsqrt_2 = torch.rsqrt(add_6) + add_6 = None + hidden_states_11 = hidden_states_10 * rsqrt_2 + hidden_states_10 = rsqrt_2 = None + to_9 = hidden_states_11.to(torch.float16) + hidden_states_11 = None + hidden_states_12 = ( + l_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ + * to_9 + ) + l_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_ = ( + to_9 + ) = None + linear_7 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_3 = linear_7.view((1, 21, -1, 64)) + linear_7 = None + query_states_1 = view_3.transpose(1, 2) + view_3 = None + linear_8 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_4 = linear_8.view((1, 21, -1, 64)) + linear_8 = None + key_states_1 = view_4.transpose(1, 2) + view_4 = None + linear_9 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_12 = l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_5 = linear_9.view((1, 21, -1, 64)) + linear_9 = None + value_states_1 = view_5.transpose(1, 2) + view_5 = None + cos_4 = cos_2.unsqueeze(1) + sin_4 = sin_2.unsqueeze(1) + mul_13 = query_states_1 * cos_4 + x1_2 = query_states_1[(Ellipsis, slice(None, 32, None))] + x2_2 = query_states_1[(Ellipsis, slice(32, None, None))] + query_states_1 = None + neg_2 = -x2_2 + x2_2 = None + cat_3 = torch.cat((neg_2, x1_2), dim=-1) + neg_2 = x1_2 = None + mul_14 = cat_3 * sin_4 + cat_3 = None + q_embed_1 = mul_13 + mul_14 + mul_13 = mul_14 = None + mul_15 = key_states_1 * cos_4 + cos_4 = None + x1_3 = key_states_1[(Ellipsis, slice(None, 32, None))] + x2_3 = key_states_1[(Ellipsis, slice(32, None, None))] + key_states_1 = None + neg_3 = -x2_3 + x2_3 = None + cat_4 = torch.cat((neg_3, x1_3), dim=-1) + neg_3 = x1_3 = None + mul_16 = cat_4 * sin_4 + cat_4 = sin_4 = None + k_embed_1 = mul_15 + mul_16 + mul_15 = mul_16 = None + getitem_13 = k_embed_1[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_1 = None + hidden_states_13 = getitem_13.expand(1, 4, 8, 21, 64) + getitem_13 = None + key_2 = hidden_states_13.reshape(1, 32, 21, 64) + hidden_states_13 = None + getitem_14 = value_states_1[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_1 = None + hidden_states_14 = getitem_14.expand(1, 4, 8, 21, 64) + getitem_14 = None + value_2 = hidden_states_14.reshape(1, 32, 21, 64) + hidden_states_14 = None + attention_mask_2 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_1 = q_embed_1.contiguous() + q_embed_1 = None + key_3 = key_2.contiguous() + key_2 = None + value_3 = value_2.contiguous() + value_2 = None + attn_output_4 = torch._C._nn.scaled_dot_product_attention( + query_1, + key_3, + value_3, + attn_mask=attention_mask_2, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_1 = key_3 = value_3 = attention_mask_2 = None + transpose_8 = attn_output_4.transpose(1, 2) + attn_output_4 = None + attn_output_5 = transpose_8.contiguous() + transpose_8 = None + reshape_5 = attn_output_5.reshape(1, 21, -1) + attn_output_5 = None + attn_output_6 = reshape_5.contiguous() + reshape_5 = None + attn_output_7 = torch._C._nn.linear( + attn_output_6, + l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_6 = l_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_15 = hidden_states_9 + attn_output_7 + hidden_states_9 = attn_output_7 = None + hidden_states_16 = hidden_states_15.to(torch.float32) + pow_4 = hidden_states_16.pow(2) + variance_3 = pow_4.mean(-1, keepdim=True) + pow_4 = None + add_10 = variance_3 + 1e-05 + variance_3 = None + rsqrt_3 = torch.rsqrt(add_10) + add_10 = None + hidden_states_17 = hidden_states_16 * rsqrt_3 + hidden_states_16 = rsqrt_3 = None + to_11 = hidden_states_17.to(torch.float16) + hidden_states_17 = None + hidden_states_18 = ( + l_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ + * to_11 + ) + l_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_ = ( + to_11 + ) = None + linear_11 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_1 = torch.nn.functional.silu(linear_11, inplace=False) + linear_11 = None + linear_12 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_18 = l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_19 = silu_1 * linear_12 + silu_1 = linear_12 = None + down_proj_1 = torch._C._nn.linear( + mul_19, + l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_19 = l_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_19 = hidden_states_15 + down_proj_1 + hidden_states_15 = down_proj_1 = None + hidden_states_20 = hidden_states_19.to(torch.float32) + pow_5 = hidden_states_20.pow(2) + variance_4 = pow_5.mean(-1, keepdim=True) + pow_5 = None + add_12 = variance_4 + 1e-05 + variance_4 = None + rsqrt_4 = torch.rsqrt(add_12) + add_12 = None + hidden_states_21 = hidden_states_20 * rsqrt_4 + hidden_states_20 = rsqrt_4 = None + to_13 = hidden_states_21.to(torch.float16) + hidden_states_21 = None + hidden_states_22 = ( + l_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ + * to_13 + ) + l_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_ = ( + to_13 + ) = None + linear_14 = torch._C._nn.linear( + hidden_states_22, + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_6 = linear_14.view((1, 21, -1, 64)) + linear_14 = None + query_states_2 = view_6.transpose(1, 2) + view_6 = None + linear_15 = torch._C._nn.linear( + hidden_states_22, + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_7 = linear_15.view((1, 21, -1, 64)) + linear_15 = None + key_states_2 = view_7.transpose(1, 2) + view_7 = None + linear_16 = torch._C._nn.linear( + hidden_states_22, + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_22 = l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_8 = linear_16.view((1, 21, -1, 64)) + linear_16 = None + value_states_2 = view_8.transpose(1, 2) + view_8 = None + cos_5 = cos_2.unsqueeze(1) + sin_5 = sin_2.unsqueeze(1) + mul_22 = query_states_2 * cos_5 + x1_4 = query_states_2[(Ellipsis, slice(None, 32, None))] + x2_4 = query_states_2[(Ellipsis, slice(32, None, None))] + query_states_2 = None + neg_4 = -x2_4 + x2_4 = None + cat_5 = torch.cat((neg_4, x1_4), dim=-1) + neg_4 = x1_4 = None + mul_23 = cat_5 * sin_5 + cat_5 = None + q_embed_2 = mul_22 + mul_23 + mul_22 = mul_23 = None + mul_24 = key_states_2 * cos_5 + cos_5 = None + x1_5 = key_states_2[(Ellipsis, slice(None, 32, None))] + x2_5 = key_states_2[(Ellipsis, slice(32, None, None))] + key_states_2 = None + neg_5 = -x2_5 + x2_5 = None + cat_6 = torch.cat((neg_5, x1_5), dim=-1) + neg_5 = x1_5 = None + mul_25 = cat_6 * sin_5 + cat_6 = sin_5 = None + k_embed_2 = mul_24 + mul_25 + mul_24 = mul_25 = None + getitem_20 = k_embed_2[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_2 = None + hidden_states_23 = getitem_20.expand(1, 4, 8, 21, 64) + getitem_20 = None + key_4 = hidden_states_23.reshape(1, 32, 21, 64) + hidden_states_23 = None + getitem_21 = value_states_2[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_2 = None + hidden_states_24 = getitem_21.expand(1, 4, 8, 21, 64) + getitem_21 = None + value_4 = hidden_states_24.reshape(1, 32, 21, 64) + hidden_states_24 = None + attention_mask_3 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_2 = q_embed_2.contiguous() + q_embed_2 = None + key_5 = key_4.contiguous() + key_4 = None + value_5 = value_4.contiguous() + value_4 = None + attn_output_8 = torch._C._nn.scaled_dot_product_attention( + query_2, + key_5, + value_5, + attn_mask=attention_mask_3, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_2 = key_5 = value_5 = attention_mask_3 = None + transpose_12 = attn_output_8.transpose(1, 2) + attn_output_8 = None + attn_output_9 = transpose_12.contiguous() + transpose_12 = None + reshape_8 = attn_output_9.reshape(1, 21, -1) + attn_output_9 = None + attn_output_10 = reshape_8.contiguous() + reshape_8 = None + attn_output_11 = torch._C._nn.linear( + attn_output_10, + l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_10 = l_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_25 = hidden_states_19 + attn_output_11 + hidden_states_19 = attn_output_11 = None + hidden_states_26 = hidden_states_25.to(torch.float32) + pow_6 = hidden_states_26.pow(2) + variance_5 = pow_6.mean(-1, keepdim=True) + pow_6 = None + add_16 = variance_5 + 1e-05 + variance_5 = None + rsqrt_5 = torch.rsqrt(add_16) + add_16 = None + hidden_states_27 = hidden_states_26 * rsqrt_5 + hidden_states_26 = rsqrt_5 = None + to_15 = hidden_states_27.to(torch.float16) + hidden_states_27 = None + hidden_states_28 = ( + l_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ + * to_15 + ) + l_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_ = ( + to_15 + ) = None + linear_18 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_2 = torch.nn.functional.silu(linear_18, inplace=False) + linear_18 = None + linear_19 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_28 = l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_28 = silu_2 * linear_19 + silu_2 = linear_19 = None + down_proj_2 = torch._C._nn.linear( + mul_28, + l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_28 = l_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_29 = hidden_states_25 + down_proj_2 + hidden_states_25 = down_proj_2 = None + hidden_states_30 = hidden_states_29.to(torch.float32) + pow_7 = hidden_states_30.pow(2) + variance_6 = pow_7.mean(-1, keepdim=True) + pow_7 = None + add_18 = variance_6 + 1e-05 + variance_6 = None + rsqrt_6 = torch.rsqrt(add_18) + add_18 = None + hidden_states_31 = hidden_states_30 * rsqrt_6 + hidden_states_30 = rsqrt_6 = None + to_17 = hidden_states_31.to(torch.float16) + hidden_states_31 = None + hidden_states_32 = ( + l_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ + * to_17 + ) + l_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_ = ( + to_17 + ) = None + linear_21 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_9 = linear_21.view((1, 21, -1, 64)) + linear_21 = None + query_states_3 = view_9.transpose(1, 2) + view_9 = None + linear_22 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_10 = linear_22.view((1, 21, -1, 64)) + linear_22 = None + key_states_3 = view_10.transpose(1, 2) + view_10 = None + linear_23 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_32 = l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_11 = linear_23.view((1, 21, -1, 64)) + linear_23 = None + value_states_3 = view_11.transpose(1, 2) + view_11 = None + cos_6 = cos_2.unsqueeze(1) + sin_6 = sin_2.unsqueeze(1) + mul_31 = query_states_3 * cos_6 + x1_6 = query_states_3[(Ellipsis, slice(None, 32, None))] + x2_6 = query_states_3[(Ellipsis, slice(32, None, None))] + query_states_3 = None + neg_6 = -x2_6 + x2_6 = None + cat_7 = torch.cat((neg_6, x1_6), dim=-1) + neg_6 = x1_6 = None + mul_32 = cat_7 * sin_6 + cat_7 = None + q_embed_3 = mul_31 + mul_32 + mul_31 = mul_32 = None + mul_33 = key_states_3 * cos_6 + cos_6 = None + x1_7 = key_states_3[(Ellipsis, slice(None, 32, None))] + x2_7 = key_states_3[(Ellipsis, slice(32, None, None))] + key_states_3 = None + neg_7 = -x2_7 + x2_7 = None + cat_8 = torch.cat((neg_7, x1_7), dim=-1) + neg_7 = x1_7 = None + mul_34 = cat_8 * sin_6 + cat_8 = sin_6 = None + k_embed_3 = mul_33 + mul_34 + mul_33 = mul_34 = None + getitem_27 = k_embed_3[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_3 = None + hidden_states_33 = getitem_27.expand(1, 4, 8, 21, 64) + getitem_27 = None + key_6 = hidden_states_33.reshape(1, 32, 21, 64) + hidden_states_33 = None + getitem_28 = value_states_3[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_3 = None + hidden_states_34 = getitem_28.expand(1, 4, 8, 21, 64) + getitem_28 = None + value_6 = hidden_states_34.reshape(1, 32, 21, 64) + hidden_states_34 = None + attention_mask_4 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_3 = q_embed_3.contiguous() + q_embed_3 = None + key_7 = key_6.contiguous() + key_6 = None + value_7 = value_6.contiguous() + value_6 = None + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_3, + key_7, + value_7, + attn_mask=attention_mask_4, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_3 = key_7 = value_7 = attention_mask_4 = None + transpose_16 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_13 = transpose_16.contiguous() + transpose_16 = None + reshape_11 = attn_output_13.reshape(1, 21, -1) + attn_output_13 = None + attn_output_14 = reshape_11.contiguous() + reshape_11 = None + attn_output_15 = torch._C._nn.linear( + attn_output_14, + l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_14 = l_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_35 = hidden_states_29 + attn_output_15 + hidden_states_29 = attn_output_15 = None + hidden_states_36 = hidden_states_35.to(torch.float32) + pow_8 = hidden_states_36.pow(2) + variance_7 = pow_8.mean(-1, keepdim=True) + pow_8 = None + add_22 = variance_7 + 1e-05 + variance_7 = None + rsqrt_7 = torch.rsqrt(add_22) + add_22 = None + hidden_states_37 = hidden_states_36 * rsqrt_7 + hidden_states_36 = rsqrt_7 = None + to_19 = hidden_states_37.to(torch.float16) + hidden_states_37 = None + hidden_states_38 = ( + l_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ + * to_19 + ) + l_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_ = ( + to_19 + ) = None + linear_25 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_3 = torch.nn.functional.silu(linear_25, inplace=False) + linear_25 = None + linear_26 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_38 = l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_37 = silu_3 * linear_26 + silu_3 = linear_26 = None + down_proj_3 = torch._C._nn.linear( + mul_37, + l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_37 = l_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_39 = hidden_states_35 + down_proj_3 + hidden_states_35 = down_proj_3 = None + hidden_states_40 = hidden_states_39.to(torch.float32) + pow_9 = hidden_states_40.pow(2) + variance_8 = pow_9.mean(-1, keepdim=True) + pow_9 = None + add_24 = variance_8 + 1e-05 + variance_8 = None + rsqrt_8 = torch.rsqrt(add_24) + add_24 = None + hidden_states_41 = hidden_states_40 * rsqrt_8 + hidden_states_40 = rsqrt_8 = None + to_21 = hidden_states_41.to(torch.float16) + hidden_states_41 = None + hidden_states_42 = ( + l_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ + * to_21 + ) + l_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_ = ( + to_21 + ) = None + linear_28 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_12 = linear_28.view((1, 21, -1, 64)) + linear_28 = None + query_states_4 = view_12.transpose(1, 2) + view_12 = None + linear_29 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_13 = linear_29.view((1, 21, -1, 64)) + linear_29 = None + key_states_4 = view_13.transpose(1, 2) + view_13 = None + linear_30 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_42 = l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_14 = linear_30.view((1, 21, -1, 64)) + linear_30 = None + value_states_4 = view_14.transpose(1, 2) + view_14 = None + cos_7 = cos_2.unsqueeze(1) + sin_7 = sin_2.unsqueeze(1) + mul_40 = query_states_4 * cos_7 + x1_8 = query_states_4[(Ellipsis, slice(None, 32, None))] + x2_8 = query_states_4[(Ellipsis, slice(32, None, None))] + query_states_4 = None + neg_8 = -x2_8 + x2_8 = None + cat_9 = torch.cat((neg_8, x1_8), dim=-1) + neg_8 = x1_8 = None + mul_41 = cat_9 * sin_7 + cat_9 = None + q_embed_4 = mul_40 + mul_41 + mul_40 = mul_41 = None + mul_42 = key_states_4 * cos_7 + cos_7 = None + x1_9 = key_states_4[(Ellipsis, slice(None, 32, None))] + x2_9 = key_states_4[(Ellipsis, slice(32, None, None))] + key_states_4 = None + neg_9 = -x2_9 + x2_9 = None + cat_10 = torch.cat((neg_9, x1_9), dim=-1) + neg_9 = x1_9 = None + mul_43 = cat_10 * sin_7 + cat_10 = sin_7 = None + k_embed_4 = mul_42 + mul_43 + mul_42 = mul_43 = None + getitem_34 = k_embed_4[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_4 = None + hidden_states_43 = getitem_34.expand(1, 4, 8, 21, 64) + getitem_34 = None + key_8 = hidden_states_43.reshape(1, 32, 21, 64) + hidden_states_43 = None + getitem_35 = value_states_4[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_4 = None + hidden_states_44 = getitem_35.expand(1, 4, 8, 21, 64) + getitem_35 = None + value_8 = hidden_states_44.reshape(1, 32, 21, 64) + hidden_states_44 = None + attention_mask_5 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_4 = q_embed_4.contiguous() + q_embed_4 = None + key_9 = key_8.contiguous() + key_8 = None + value_9 = value_8.contiguous() + value_8 = None + attn_output_16 = torch._C._nn.scaled_dot_product_attention( + query_4, + key_9, + value_9, + attn_mask=attention_mask_5, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_4 = key_9 = value_9 = attention_mask_5 = None + transpose_20 = attn_output_16.transpose(1, 2) + attn_output_16 = None + attn_output_17 = transpose_20.contiguous() + transpose_20 = None + reshape_14 = attn_output_17.reshape(1, 21, -1) + attn_output_17 = None + attn_output_18 = reshape_14.contiguous() + reshape_14 = None + attn_output_19 = torch._C._nn.linear( + attn_output_18, + l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_18 = l_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_45 = hidden_states_39 + attn_output_19 + hidden_states_39 = attn_output_19 = None + hidden_states_46 = hidden_states_45.to(torch.float32) + pow_10 = hidden_states_46.pow(2) + variance_9 = pow_10.mean(-1, keepdim=True) + pow_10 = None + add_28 = variance_9 + 1e-05 + variance_9 = None + rsqrt_9 = torch.rsqrt(add_28) + add_28 = None + hidden_states_47 = hidden_states_46 * rsqrt_9 + hidden_states_46 = rsqrt_9 = None + to_23 = hidden_states_47.to(torch.float16) + hidden_states_47 = None + hidden_states_48 = ( + l_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ + * to_23 + ) + l_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_ = ( + to_23 + ) = None + linear_32 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_4 = torch.nn.functional.silu(linear_32, inplace=False) + linear_32 = None + linear_33 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_48 = l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_46 = silu_4 * linear_33 + silu_4 = linear_33 = None + down_proj_4 = torch._C._nn.linear( + mul_46, + l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_46 = l_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_49 = hidden_states_45 + down_proj_4 + hidden_states_45 = down_proj_4 = None + hidden_states_50 = hidden_states_49.to(torch.float32) + pow_11 = hidden_states_50.pow(2) + variance_10 = pow_11.mean(-1, keepdim=True) + pow_11 = None + add_30 = variance_10 + 1e-05 + variance_10 = None + rsqrt_10 = torch.rsqrt(add_30) + add_30 = None + hidden_states_51 = hidden_states_50 * rsqrt_10 + hidden_states_50 = rsqrt_10 = None + to_25 = hidden_states_51.to(torch.float16) + hidden_states_51 = None + hidden_states_52 = ( + l_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ + * to_25 + ) + l_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_ = ( + to_25 + ) = None + linear_35 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_15 = linear_35.view((1, 21, -1, 64)) + linear_35 = None + query_states_5 = view_15.transpose(1, 2) + view_15 = None + linear_36 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_16 = linear_36.view((1, 21, -1, 64)) + linear_36 = None + key_states_5 = view_16.transpose(1, 2) + view_16 = None + linear_37 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_52 = l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_17 = linear_37.view((1, 21, -1, 64)) + linear_37 = None + value_states_5 = view_17.transpose(1, 2) + view_17 = None + cos_8 = cos_2.unsqueeze(1) + sin_8 = sin_2.unsqueeze(1) + mul_49 = query_states_5 * cos_8 + x1_10 = query_states_5[(Ellipsis, slice(None, 32, None))] + x2_10 = query_states_5[(Ellipsis, slice(32, None, None))] + query_states_5 = None + neg_10 = -x2_10 + x2_10 = None + cat_11 = torch.cat((neg_10, x1_10), dim=-1) + neg_10 = x1_10 = None + mul_50 = cat_11 * sin_8 + cat_11 = None + q_embed_5 = mul_49 + mul_50 + mul_49 = mul_50 = None + mul_51 = key_states_5 * cos_8 + cos_8 = None + x1_11 = key_states_5[(Ellipsis, slice(None, 32, None))] + x2_11 = key_states_5[(Ellipsis, slice(32, None, None))] + key_states_5 = None + neg_11 = -x2_11 + x2_11 = None + cat_12 = torch.cat((neg_11, x1_11), dim=-1) + neg_11 = x1_11 = None + mul_52 = cat_12 * sin_8 + cat_12 = sin_8 = None + k_embed_5 = mul_51 + mul_52 + mul_51 = mul_52 = None + getitem_41 = k_embed_5[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_5 = None + hidden_states_53 = getitem_41.expand(1, 4, 8, 21, 64) + getitem_41 = None + key_10 = hidden_states_53.reshape(1, 32, 21, 64) + hidden_states_53 = None + getitem_42 = value_states_5[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_5 = None + hidden_states_54 = getitem_42.expand(1, 4, 8, 21, 64) + getitem_42 = None + value_10 = hidden_states_54.reshape(1, 32, 21, 64) + hidden_states_54 = None + attention_mask_6 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_5 = q_embed_5.contiguous() + q_embed_5 = None + key_11 = key_10.contiguous() + key_10 = None + value_11 = value_10.contiguous() + value_10 = None + attn_output_20 = torch._C._nn.scaled_dot_product_attention( + query_5, + key_11, + value_11, + attn_mask=attention_mask_6, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_5 = key_11 = value_11 = attention_mask_6 = None + transpose_24 = attn_output_20.transpose(1, 2) + attn_output_20 = None + attn_output_21 = transpose_24.contiguous() + transpose_24 = None + reshape_17 = attn_output_21.reshape(1, 21, -1) + attn_output_21 = None + attn_output_22 = reshape_17.contiguous() + reshape_17 = None + attn_output_23 = torch._C._nn.linear( + attn_output_22, + l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_22 = l_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_55 = hidden_states_49 + attn_output_23 + hidden_states_49 = attn_output_23 = None + hidden_states_56 = hidden_states_55.to(torch.float32) + pow_12 = hidden_states_56.pow(2) + variance_11 = pow_12.mean(-1, keepdim=True) + pow_12 = None + add_34 = variance_11 + 1e-05 + variance_11 = None + rsqrt_11 = torch.rsqrt(add_34) + add_34 = None + hidden_states_57 = hidden_states_56 * rsqrt_11 + hidden_states_56 = rsqrt_11 = None + to_27 = hidden_states_57.to(torch.float16) + hidden_states_57 = None + hidden_states_58 = ( + l_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ + * to_27 + ) + l_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_ = ( + to_27 + ) = None + linear_39 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_5 = torch.nn.functional.silu(linear_39, inplace=False) + linear_39 = None + linear_40 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_58 = l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_55 = silu_5 * linear_40 + silu_5 = linear_40 = None + down_proj_5 = torch._C._nn.linear( + mul_55, + l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_55 = l_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_59 = hidden_states_55 + down_proj_5 + hidden_states_55 = down_proj_5 = None + hidden_states_60 = hidden_states_59.to(torch.float32) + pow_13 = hidden_states_60.pow(2) + variance_12 = pow_13.mean(-1, keepdim=True) + pow_13 = None + add_36 = variance_12 + 1e-05 + variance_12 = None + rsqrt_12 = torch.rsqrt(add_36) + add_36 = None + hidden_states_61 = hidden_states_60 * rsqrt_12 + hidden_states_60 = rsqrt_12 = None + to_29 = hidden_states_61.to(torch.float16) + hidden_states_61 = None + hidden_states_62 = ( + l_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ + * to_29 + ) + l_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_ = ( + to_29 + ) = None + linear_42 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_18 = linear_42.view((1, 21, -1, 64)) + linear_42 = None + query_states_6 = view_18.transpose(1, 2) + view_18 = None + linear_43 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_19 = linear_43.view((1, 21, -1, 64)) + linear_43 = None + key_states_6 = view_19.transpose(1, 2) + view_19 = None + linear_44 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_62 = l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_20 = linear_44.view((1, 21, -1, 64)) + linear_44 = None + value_states_6 = view_20.transpose(1, 2) + view_20 = None + cos_9 = cos_2.unsqueeze(1) + sin_9 = sin_2.unsqueeze(1) + mul_58 = query_states_6 * cos_9 + x1_12 = query_states_6[(Ellipsis, slice(None, 32, None))] + x2_12 = query_states_6[(Ellipsis, slice(32, None, None))] + query_states_6 = None + neg_12 = -x2_12 + x2_12 = None + cat_13 = torch.cat((neg_12, x1_12), dim=-1) + neg_12 = x1_12 = None + mul_59 = cat_13 * sin_9 + cat_13 = None + q_embed_6 = mul_58 + mul_59 + mul_58 = mul_59 = None + mul_60 = key_states_6 * cos_9 + cos_9 = None + x1_13 = key_states_6[(Ellipsis, slice(None, 32, None))] + x2_13 = key_states_6[(Ellipsis, slice(32, None, None))] + key_states_6 = None + neg_13 = -x2_13 + x2_13 = None + cat_14 = torch.cat((neg_13, x1_13), dim=-1) + neg_13 = x1_13 = None + mul_61 = cat_14 * sin_9 + cat_14 = sin_9 = None + k_embed_6 = mul_60 + mul_61 + mul_60 = mul_61 = None + getitem_48 = k_embed_6[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_6 = None + hidden_states_63 = getitem_48.expand(1, 4, 8, 21, 64) + getitem_48 = None + key_12 = hidden_states_63.reshape(1, 32, 21, 64) + hidden_states_63 = None + getitem_49 = value_states_6[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_6 = None + hidden_states_64 = getitem_49.expand(1, 4, 8, 21, 64) + getitem_49 = None + value_12 = hidden_states_64.reshape(1, 32, 21, 64) + hidden_states_64 = None + attention_mask_7 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_6 = q_embed_6.contiguous() + q_embed_6 = None + key_13 = key_12.contiguous() + key_12 = None + value_13 = value_12.contiguous() + value_12 = None + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_6, + key_13, + value_13, + attn_mask=attention_mask_7, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_6 = key_13 = value_13 = attention_mask_7 = None + transpose_28 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_25 = transpose_28.contiguous() + transpose_28 = None + reshape_20 = attn_output_25.reshape(1, 21, -1) + attn_output_25 = None + attn_output_26 = reshape_20.contiguous() + reshape_20 = None + attn_output_27 = torch._C._nn.linear( + attn_output_26, + l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_26 = l_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_65 = hidden_states_59 + attn_output_27 + hidden_states_59 = attn_output_27 = None + hidden_states_66 = hidden_states_65.to(torch.float32) + pow_14 = hidden_states_66.pow(2) + variance_13 = pow_14.mean(-1, keepdim=True) + pow_14 = None + add_40 = variance_13 + 1e-05 + variance_13 = None + rsqrt_13 = torch.rsqrt(add_40) + add_40 = None + hidden_states_67 = hidden_states_66 * rsqrt_13 + hidden_states_66 = rsqrt_13 = None + to_31 = hidden_states_67.to(torch.float16) + hidden_states_67 = None + hidden_states_68 = ( + l_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ + * to_31 + ) + l_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_ = ( + to_31 + ) = None + linear_46 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_6 = torch.nn.functional.silu(linear_46, inplace=False) + linear_46 = None + linear_47 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_68 = l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_64 = silu_6 * linear_47 + silu_6 = linear_47 = None + down_proj_6 = torch._C._nn.linear( + mul_64, + l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_64 = l_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_69 = hidden_states_65 + down_proj_6 + hidden_states_65 = down_proj_6 = None + hidden_states_70 = hidden_states_69.to(torch.float32) + pow_15 = hidden_states_70.pow(2) + variance_14 = pow_15.mean(-1, keepdim=True) + pow_15 = None + add_42 = variance_14 + 1e-05 + variance_14 = None + rsqrt_14 = torch.rsqrt(add_42) + add_42 = None + hidden_states_71 = hidden_states_70 * rsqrt_14 + hidden_states_70 = rsqrt_14 = None + to_33 = hidden_states_71.to(torch.float16) + hidden_states_71 = None + hidden_states_72 = ( + l_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ + * to_33 + ) + l_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_ = ( + to_33 + ) = None + linear_49 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_21 = linear_49.view((1, 21, -1, 64)) + linear_49 = None + query_states_7 = view_21.transpose(1, 2) + view_21 = None + linear_50 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_22 = linear_50.view((1, 21, -1, 64)) + linear_50 = None + key_states_7 = view_22.transpose(1, 2) + view_22 = None + linear_51 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_72 = l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_23 = linear_51.view((1, 21, -1, 64)) + linear_51 = None + value_states_7 = view_23.transpose(1, 2) + view_23 = None + cos_10 = cos_2.unsqueeze(1) + sin_10 = sin_2.unsqueeze(1) + mul_67 = query_states_7 * cos_10 + x1_14 = query_states_7[(Ellipsis, slice(None, 32, None))] + x2_14 = query_states_7[(Ellipsis, slice(32, None, None))] + query_states_7 = None + neg_14 = -x2_14 + x2_14 = None + cat_15 = torch.cat((neg_14, x1_14), dim=-1) + neg_14 = x1_14 = None + mul_68 = cat_15 * sin_10 + cat_15 = None + q_embed_7 = mul_67 + mul_68 + mul_67 = mul_68 = None + mul_69 = key_states_7 * cos_10 + cos_10 = None + x1_15 = key_states_7[(Ellipsis, slice(None, 32, None))] + x2_15 = key_states_7[(Ellipsis, slice(32, None, None))] + key_states_7 = None + neg_15 = -x2_15 + x2_15 = None + cat_16 = torch.cat((neg_15, x1_15), dim=-1) + neg_15 = x1_15 = None + mul_70 = cat_16 * sin_10 + cat_16 = sin_10 = None + k_embed_7 = mul_69 + mul_70 + mul_69 = mul_70 = None + getitem_55 = k_embed_7[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_7 = None + hidden_states_73 = getitem_55.expand(1, 4, 8, 21, 64) + getitem_55 = None + key_14 = hidden_states_73.reshape(1, 32, 21, 64) + hidden_states_73 = None + getitem_56 = value_states_7[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_7 = None + hidden_states_74 = getitem_56.expand(1, 4, 8, 21, 64) + getitem_56 = None + value_14 = hidden_states_74.reshape(1, 32, 21, 64) + hidden_states_74 = None + attention_mask_8 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_7 = q_embed_7.contiguous() + q_embed_7 = None + key_15 = key_14.contiguous() + key_14 = None + value_15 = value_14.contiguous() + value_14 = None + attn_output_28 = torch._C._nn.scaled_dot_product_attention( + query_7, + key_15, + value_15, + attn_mask=attention_mask_8, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_7 = key_15 = value_15 = attention_mask_8 = None + transpose_32 = attn_output_28.transpose(1, 2) + attn_output_28 = None + attn_output_29 = transpose_32.contiguous() + transpose_32 = None + reshape_23 = attn_output_29.reshape(1, 21, -1) + attn_output_29 = None + attn_output_30 = reshape_23.contiguous() + reshape_23 = None + attn_output_31 = torch._C._nn.linear( + attn_output_30, + l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_30 = l_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_75 = hidden_states_69 + attn_output_31 + hidden_states_69 = attn_output_31 = None + hidden_states_76 = hidden_states_75.to(torch.float32) + pow_16 = hidden_states_76.pow(2) + variance_15 = pow_16.mean(-1, keepdim=True) + pow_16 = None + add_46 = variance_15 + 1e-05 + variance_15 = None + rsqrt_15 = torch.rsqrt(add_46) + add_46 = None + hidden_states_77 = hidden_states_76 * rsqrt_15 + hidden_states_76 = rsqrt_15 = None + to_35 = hidden_states_77.to(torch.float16) + hidden_states_77 = None + hidden_states_78 = ( + l_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ + * to_35 + ) + l_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_ = ( + to_35 + ) = None + linear_53 = torch._C._nn.linear( + hidden_states_78, + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_7 = torch.nn.functional.silu(linear_53, inplace=False) + linear_53 = None + linear_54 = torch._C._nn.linear( + hidden_states_78, + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_78 = l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_73 = silu_7 * linear_54 + silu_7 = linear_54 = None + down_proj_7 = torch._C._nn.linear( + mul_73, + l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_73 = l_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_79 = hidden_states_75 + down_proj_7 + hidden_states_75 = down_proj_7 = None + hidden_states_80 = hidden_states_79.to(torch.float32) + pow_17 = hidden_states_80.pow(2) + variance_16 = pow_17.mean(-1, keepdim=True) + pow_17 = None + add_48 = variance_16 + 1e-05 + variance_16 = None + rsqrt_16 = torch.rsqrt(add_48) + add_48 = None + hidden_states_81 = hidden_states_80 * rsqrt_16 + hidden_states_80 = rsqrt_16 = None + to_37 = hidden_states_81.to(torch.float16) + hidden_states_81 = None + hidden_states_82 = ( + l_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ + * to_37 + ) + l_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_ = ( + to_37 + ) = None + linear_56 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_24 = linear_56.view((1, 21, -1, 64)) + linear_56 = None + query_states_8 = view_24.transpose(1, 2) + view_24 = None + linear_57 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_25 = linear_57.view((1, 21, -1, 64)) + linear_57 = None + key_states_8 = view_25.transpose(1, 2) + view_25 = None + linear_58 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_82 = l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_26 = linear_58.view((1, 21, -1, 64)) + linear_58 = None + value_states_8 = view_26.transpose(1, 2) + view_26 = None + cos_11 = cos_2.unsqueeze(1) + sin_11 = sin_2.unsqueeze(1) + mul_76 = query_states_8 * cos_11 + x1_16 = query_states_8[(Ellipsis, slice(None, 32, None))] + x2_16 = query_states_8[(Ellipsis, slice(32, None, None))] + query_states_8 = None + neg_16 = -x2_16 + x2_16 = None + cat_17 = torch.cat((neg_16, x1_16), dim=-1) + neg_16 = x1_16 = None + mul_77 = cat_17 * sin_11 + cat_17 = None + q_embed_8 = mul_76 + mul_77 + mul_76 = mul_77 = None + mul_78 = key_states_8 * cos_11 + cos_11 = None + x1_17 = key_states_8[(Ellipsis, slice(None, 32, None))] + x2_17 = key_states_8[(Ellipsis, slice(32, None, None))] + key_states_8 = None + neg_17 = -x2_17 + x2_17 = None + cat_18 = torch.cat((neg_17, x1_17), dim=-1) + neg_17 = x1_17 = None + mul_79 = cat_18 * sin_11 + cat_18 = sin_11 = None + k_embed_8 = mul_78 + mul_79 + mul_78 = mul_79 = None + getitem_62 = k_embed_8[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_8 = None + hidden_states_83 = getitem_62.expand(1, 4, 8, 21, 64) + getitem_62 = None + key_16 = hidden_states_83.reshape(1, 32, 21, 64) + hidden_states_83 = None + getitem_63 = value_states_8[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_8 = None + hidden_states_84 = getitem_63.expand(1, 4, 8, 21, 64) + getitem_63 = None + value_16 = hidden_states_84.reshape(1, 32, 21, 64) + hidden_states_84 = None + attention_mask_9 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_8 = q_embed_8.contiguous() + q_embed_8 = None + key_17 = key_16.contiguous() + key_16 = None + value_17 = value_16.contiguous() + value_16 = None + attn_output_32 = torch._C._nn.scaled_dot_product_attention( + query_8, + key_17, + value_17, + attn_mask=attention_mask_9, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_8 = key_17 = value_17 = attention_mask_9 = None + transpose_36 = attn_output_32.transpose(1, 2) + attn_output_32 = None + attn_output_33 = transpose_36.contiguous() + transpose_36 = None + reshape_26 = attn_output_33.reshape(1, 21, -1) + attn_output_33 = None + attn_output_34 = reshape_26.contiguous() + reshape_26 = None + attn_output_35 = torch._C._nn.linear( + attn_output_34, + l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_34 = l_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_85 = hidden_states_79 + attn_output_35 + hidden_states_79 = attn_output_35 = None + hidden_states_86 = hidden_states_85.to(torch.float32) + pow_18 = hidden_states_86.pow(2) + variance_17 = pow_18.mean(-1, keepdim=True) + pow_18 = None + add_52 = variance_17 + 1e-05 + variance_17 = None + rsqrt_17 = torch.rsqrt(add_52) + add_52 = None + hidden_states_87 = hidden_states_86 * rsqrt_17 + hidden_states_86 = rsqrt_17 = None + to_39 = hidden_states_87.to(torch.float16) + hidden_states_87 = None + hidden_states_88 = ( + l_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ + * to_39 + ) + l_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_ = ( + to_39 + ) = None + linear_60 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_8 = torch.nn.functional.silu(linear_60, inplace=False) + linear_60 = None + linear_61 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_88 = l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_82 = silu_8 * linear_61 + silu_8 = linear_61 = None + down_proj_8 = torch._C._nn.linear( + mul_82, + l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_82 = l_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_89 = hidden_states_85 + down_proj_8 + hidden_states_85 = down_proj_8 = None + hidden_states_90 = hidden_states_89.to(torch.float32) + pow_19 = hidden_states_90.pow(2) + variance_18 = pow_19.mean(-1, keepdim=True) + pow_19 = None + add_54 = variance_18 + 1e-05 + variance_18 = None + rsqrt_18 = torch.rsqrt(add_54) + add_54 = None + hidden_states_91 = hidden_states_90 * rsqrt_18 + hidden_states_90 = rsqrt_18 = None + to_41 = hidden_states_91.to(torch.float16) + hidden_states_91 = None + hidden_states_92 = ( + l_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ + * to_41 + ) + l_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_ = ( + to_41 + ) = None + linear_63 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_27 = linear_63.view((1, 21, -1, 64)) + linear_63 = None + query_states_9 = view_27.transpose(1, 2) + view_27 = None + linear_64 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_28 = linear_64.view((1, 21, -1, 64)) + linear_64 = None + key_states_9 = view_28.transpose(1, 2) + view_28 = None + linear_65 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_92 = l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_29 = linear_65.view((1, 21, -1, 64)) + linear_65 = None + value_states_9 = view_29.transpose(1, 2) + view_29 = None + cos_12 = cos_2.unsqueeze(1) + sin_12 = sin_2.unsqueeze(1) + mul_85 = query_states_9 * cos_12 + x1_18 = query_states_9[(Ellipsis, slice(None, 32, None))] + x2_18 = query_states_9[(Ellipsis, slice(32, None, None))] + query_states_9 = None + neg_18 = -x2_18 + x2_18 = None + cat_19 = torch.cat((neg_18, x1_18), dim=-1) + neg_18 = x1_18 = None + mul_86 = cat_19 * sin_12 + cat_19 = None + q_embed_9 = mul_85 + mul_86 + mul_85 = mul_86 = None + mul_87 = key_states_9 * cos_12 + cos_12 = None + x1_19 = key_states_9[(Ellipsis, slice(None, 32, None))] + x2_19 = key_states_9[(Ellipsis, slice(32, None, None))] + key_states_9 = None + neg_19 = -x2_19 + x2_19 = None + cat_20 = torch.cat((neg_19, x1_19), dim=-1) + neg_19 = x1_19 = None + mul_88 = cat_20 * sin_12 + cat_20 = sin_12 = None + k_embed_9 = mul_87 + mul_88 + mul_87 = mul_88 = None + getitem_69 = k_embed_9[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_9 = None + hidden_states_93 = getitem_69.expand(1, 4, 8, 21, 64) + getitem_69 = None + key_18 = hidden_states_93.reshape(1, 32, 21, 64) + hidden_states_93 = None + getitem_70 = value_states_9[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_9 = None + hidden_states_94 = getitem_70.expand(1, 4, 8, 21, 64) + getitem_70 = None + value_18 = hidden_states_94.reshape(1, 32, 21, 64) + hidden_states_94 = None + attention_mask_10 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_9 = q_embed_9.contiguous() + q_embed_9 = None + key_19 = key_18.contiguous() + key_18 = None + value_19 = value_18.contiguous() + value_18 = None + attn_output_36 = torch._C._nn.scaled_dot_product_attention( + query_9, + key_19, + value_19, + attn_mask=attention_mask_10, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_9 = key_19 = value_19 = attention_mask_10 = None + transpose_40 = attn_output_36.transpose(1, 2) + attn_output_36 = None + attn_output_37 = transpose_40.contiguous() + transpose_40 = None + reshape_29 = attn_output_37.reshape(1, 21, -1) + attn_output_37 = None + attn_output_38 = reshape_29.contiguous() + reshape_29 = None + attn_output_39 = torch._C._nn.linear( + attn_output_38, + l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_38 = l_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_95 = hidden_states_89 + attn_output_39 + hidden_states_89 = attn_output_39 = None + hidden_states_96 = hidden_states_95.to(torch.float32) + pow_20 = hidden_states_96.pow(2) + variance_19 = pow_20.mean(-1, keepdim=True) + pow_20 = None + add_58 = variance_19 + 1e-05 + variance_19 = None + rsqrt_19 = torch.rsqrt(add_58) + add_58 = None + hidden_states_97 = hidden_states_96 * rsqrt_19 + hidden_states_96 = rsqrt_19 = None + to_43 = hidden_states_97.to(torch.float16) + hidden_states_97 = None + hidden_states_98 = ( + l_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ + * to_43 + ) + l_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_ = ( + to_43 + ) = None + linear_67 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_9 = torch.nn.functional.silu(linear_67, inplace=False) + linear_67 = None + linear_68 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_98 = l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_91 = silu_9 * linear_68 + silu_9 = linear_68 = None + down_proj_9 = torch._C._nn.linear( + mul_91, + l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_91 = l_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_99 = hidden_states_95 + down_proj_9 + hidden_states_95 = down_proj_9 = None + hidden_states_100 = hidden_states_99.to(torch.float32) + pow_21 = hidden_states_100.pow(2) + variance_20 = pow_21.mean(-1, keepdim=True) + pow_21 = None + add_60 = variance_20 + 1e-05 + variance_20 = None + rsqrt_20 = torch.rsqrt(add_60) + add_60 = None + hidden_states_101 = hidden_states_100 * rsqrt_20 + hidden_states_100 = rsqrt_20 = None + to_45 = hidden_states_101.to(torch.float16) + hidden_states_101 = None + hidden_states_102 = ( + l_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ + * to_45 + ) + l_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_ = ( + to_45 + ) = None + linear_70 = torch._C._nn.linear( + hidden_states_102, + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_30 = linear_70.view((1, 21, -1, 64)) + linear_70 = None + query_states_10 = view_30.transpose(1, 2) + view_30 = None + linear_71 = torch._C._nn.linear( + hidden_states_102, + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_31 = linear_71.view((1, 21, -1, 64)) + linear_71 = None + key_states_10 = view_31.transpose(1, 2) + view_31 = None + linear_72 = torch._C._nn.linear( + hidden_states_102, + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_102 = l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_32 = linear_72.view((1, 21, -1, 64)) + linear_72 = None + value_states_10 = view_32.transpose(1, 2) + view_32 = None + cos_13 = cos_2.unsqueeze(1) + sin_13 = sin_2.unsqueeze(1) + mul_94 = query_states_10 * cos_13 + x1_20 = query_states_10[(Ellipsis, slice(None, 32, None))] + x2_20 = query_states_10[(Ellipsis, slice(32, None, None))] + query_states_10 = None + neg_20 = -x2_20 + x2_20 = None + cat_21 = torch.cat((neg_20, x1_20), dim=-1) + neg_20 = x1_20 = None + mul_95 = cat_21 * sin_13 + cat_21 = None + q_embed_10 = mul_94 + mul_95 + mul_94 = mul_95 = None + mul_96 = key_states_10 * cos_13 + cos_13 = None + x1_21 = key_states_10[(Ellipsis, slice(None, 32, None))] + x2_21 = key_states_10[(Ellipsis, slice(32, None, None))] + key_states_10 = None + neg_21 = -x2_21 + x2_21 = None + cat_22 = torch.cat((neg_21, x1_21), dim=-1) + neg_21 = x1_21 = None + mul_97 = cat_22 * sin_13 + cat_22 = sin_13 = None + k_embed_10 = mul_96 + mul_97 + mul_96 = mul_97 = None + getitem_76 = k_embed_10[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_10 = None + hidden_states_103 = getitem_76.expand(1, 4, 8, 21, 64) + getitem_76 = None + key_20 = hidden_states_103.reshape(1, 32, 21, 64) + hidden_states_103 = None + getitem_77 = value_states_10[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_10 = None + hidden_states_104 = getitem_77.expand(1, 4, 8, 21, 64) + getitem_77 = None + value_20 = hidden_states_104.reshape(1, 32, 21, 64) + hidden_states_104 = None + attention_mask_11 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_10 = q_embed_10.contiguous() + q_embed_10 = None + key_21 = key_20.contiguous() + key_20 = None + value_21 = value_20.contiguous() + value_20 = None + attn_output_40 = torch._C._nn.scaled_dot_product_attention( + query_10, + key_21, + value_21, + attn_mask=attention_mask_11, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_10 = key_21 = value_21 = attention_mask_11 = None + transpose_44 = attn_output_40.transpose(1, 2) + attn_output_40 = None + attn_output_41 = transpose_44.contiguous() + transpose_44 = None + reshape_32 = attn_output_41.reshape(1, 21, -1) + attn_output_41 = None + attn_output_42 = reshape_32.contiguous() + reshape_32 = None + attn_output_43 = torch._C._nn.linear( + attn_output_42, + l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_42 = l_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_105 = hidden_states_99 + attn_output_43 + hidden_states_99 = attn_output_43 = None + hidden_states_106 = hidden_states_105.to(torch.float32) + pow_22 = hidden_states_106.pow(2) + variance_21 = pow_22.mean(-1, keepdim=True) + pow_22 = None + add_64 = variance_21 + 1e-05 + variance_21 = None + rsqrt_21 = torch.rsqrt(add_64) + add_64 = None + hidden_states_107 = hidden_states_106 * rsqrt_21 + hidden_states_106 = rsqrt_21 = None + to_47 = hidden_states_107.to(torch.float16) + hidden_states_107 = None + hidden_states_108 = ( + l_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ + * to_47 + ) + l_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_ = ( + to_47 + ) = None + linear_74 = torch._C._nn.linear( + hidden_states_108, + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_10 = torch.nn.functional.silu(linear_74, inplace=False) + linear_74 = None + linear_75 = torch._C._nn.linear( + hidden_states_108, + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_108 = l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_100 = silu_10 * linear_75 + silu_10 = linear_75 = None + down_proj_10 = torch._C._nn.linear( + mul_100, + l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_100 = l_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_109 = hidden_states_105 + down_proj_10 + hidden_states_105 = down_proj_10 = None + hidden_states_110 = hidden_states_109.to(torch.float32) + pow_23 = hidden_states_110.pow(2) + variance_22 = pow_23.mean(-1, keepdim=True) + pow_23 = None + add_66 = variance_22 + 1e-05 + variance_22 = None + rsqrt_22 = torch.rsqrt(add_66) + add_66 = None + hidden_states_111 = hidden_states_110 * rsqrt_22 + hidden_states_110 = rsqrt_22 = None + to_49 = hidden_states_111.to(torch.float16) + hidden_states_111 = None + hidden_states_112 = ( + l_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ + * to_49 + ) + l_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_ = ( + to_49 + ) = None + linear_77 = torch._C._nn.linear( + hidden_states_112, + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_33 = linear_77.view((1, 21, -1, 64)) + linear_77 = None + query_states_11 = view_33.transpose(1, 2) + view_33 = None + linear_78 = torch._C._nn.linear( + hidden_states_112, + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_34 = linear_78.view((1, 21, -1, 64)) + linear_78 = None + key_states_11 = view_34.transpose(1, 2) + view_34 = None + linear_79 = torch._C._nn.linear( + hidden_states_112, + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_112 = l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_35 = linear_79.view((1, 21, -1, 64)) + linear_79 = None + value_states_11 = view_35.transpose(1, 2) + view_35 = None + cos_14 = cos_2.unsqueeze(1) + sin_14 = sin_2.unsqueeze(1) + mul_103 = query_states_11 * cos_14 + x1_22 = query_states_11[(Ellipsis, slice(None, 32, None))] + x2_22 = query_states_11[(Ellipsis, slice(32, None, None))] + query_states_11 = None + neg_22 = -x2_22 + x2_22 = None + cat_23 = torch.cat((neg_22, x1_22), dim=-1) + neg_22 = x1_22 = None + mul_104 = cat_23 * sin_14 + cat_23 = None + q_embed_11 = mul_103 + mul_104 + mul_103 = mul_104 = None + mul_105 = key_states_11 * cos_14 + cos_14 = None + x1_23 = key_states_11[(Ellipsis, slice(None, 32, None))] + x2_23 = key_states_11[(Ellipsis, slice(32, None, None))] + key_states_11 = None + neg_23 = -x2_23 + x2_23 = None + cat_24 = torch.cat((neg_23, x1_23), dim=-1) + neg_23 = x1_23 = None + mul_106 = cat_24 * sin_14 + cat_24 = sin_14 = None + k_embed_11 = mul_105 + mul_106 + mul_105 = mul_106 = None + getitem_83 = k_embed_11[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_11 = None + hidden_states_113 = getitem_83.expand(1, 4, 8, 21, 64) + getitem_83 = None + key_22 = hidden_states_113.reshape(1, 32, 21, 64) + hidden_states_113 = None + getitem_84 = value_states_11[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_11 = None + hidden_states_114 = getitem_84.expand(1, 4, 8, 21, 64) + getitem_84 = None + value_22 = hidden_states_114.reshape(1, 32, 21, 64) + hidden_states_114 = None + attention_mask_12 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_11 = q_embed_11.contiguous() + q_embed_11 = None + key_23 = key_22.contiguous() + key_22 = None + value_23 = value_22.contiguous() + value_22 = None + attn_output_44 = torch._C._nn.scaled_dot_product_attention( + query_11, + key_23, + value_23, + attn_mask=attention_mask_12, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_11 = key_23 = value_23 = attention_mask_12 = None + transpose_48 = attn_output_44.transpose(1, 2) + attn_output_44 = None + attn_output_45 = transpose_48.contiguous() + transpose_48 = None + reshape_35 = attn_output_45.reshape(1, 21, -1) + attn_output_45 = None + attn_output_46 = reshape_35.contiguous() + reshape_35 = None + attn_output_47 = torch._C._nn.linear( + attn_output_46, + l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_46 = l_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_115 = hidden_states_109 + attn_output_47 + hidden_states_109 = attn_output_47 = None + hidden_states_116 = hidden_states_115.to(torch.float32) + pow_24 = hidden_states_116.pow(2) + variance_23 = pow_24.mean(-1, keepdim=True) + pow_24 = None + add_70 = variance_23 + 1e-05 + variance_23 = None + rsqrt_23 = torch.rsqrt(add_70) + add_70 = None + hidden_states_117 = hidden_states_116 * rsqrt_23 + hidden_states_116 = rsqrt_23 = None + to_51 = hidden_states_117.to(torch.float16) + hidden_states_117 = None + hidden_states_118 = ( + l_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ + * to_51 + ) + l_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_ = ( + to_51 + ) = None + linear_81 = torch._C._nn.linear( + hidden_states_118, + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_11 = torch.nn.functional.silu(linear_81, inplace=False) + linear_81 = None + linear_82 = torch._C._nn.linear( + hidden_states_118, + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_118 = l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_109 = silu_11 * linear_82 + silu_11 = linear_82 = None + down_proj_11 = torch._C._nn.linear( + mul_109, + l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_109 = l_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_119 = hidden_states_115 + down_proj_11 + hidden_states_115 = down_proj_11 = None + hidden_states_120 = hidden_states_119.to(torch.float32) + pow_25 = hidden_states_120.pow(2) + variance_24 = pow_25.mean(-1, keepdim=True) + pow_25 = None + add_72 = variance_24 + 1e-05 + variance_24 = None + rsqrt_24 = torch.rsqrt(add_72) + add_72 = None + hidden_states_121 = hidden_states_120 * rsqrt_24 + hidden_states_120 = rsqrt_24 = None + to_53 = hidden_states_121.to(torch.float16) + hidden_states_121 = None + hidden_states_122 = ( + l_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ + * to_53 + ) + l_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_ = ( + to_53 + ) = None + linear_84 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_36 = linear_84.view((1, 21, -1, 64)) + linear_84 = None + query_states_12 = view_36.transpose(1, 2) + view_36 = None + linear_85 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_37 = linear_85.view((1, 21, -1, 64)) + linear_85 = None + key_states_12 = view_37.transpose(1, 2) + view_37 = None + linear_86 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_122 = l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_38 = linear_86.view((1, 21, -1, 64)) + linear_86 = None + value_states_12 = view_38.transpose(1, 2) + view_38 = None + cos_15 = cos_2.unsqueeze(1) + sin_15 = sin_2.unsqueeze(1) + mul_112 = query_states_12 * cos_15 + x1_24 = query_states_12[(Ellipsis, slice(None, 32, None))] + x2_24 = query_states_12[(Ellipsis, slice(32, None, None))] + query_states_12 = None + neg_24 = -x2_24 + x2_24 = None + cat_25 = torch.cat((neg_24, x1_24), dim=-1) + neg_24 = x1_24 = None + mul_113 = cat_25 * sin_15 + cat_25 = None + q_embed_12 = mul_112 + mul_113 + mul_112 = mul_113 = None + mul_114 = key_states_12 * cos_15 + cos_15 = None + x1_25 = key_states_12[(Ellipsis, slice(None, 32, None))] + x2_25 = key_states_12[(Ellipsis, slice(32, None, None))] + key_states_12 = None + neg_25 = -x2_25 + x2_25 = None + cat_26 = torch.cat((neg_25, x1_25), dim=-1) + neg_25 = x1_25 = None + mul_115 = cat_26 * sin_15 + cat_26 = sin_15 = None + k_embed_12 = mul_114 + mul_115 + mul_114 = mul_115 = None + getitem_90 = k_embed_12[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_12 = None + hidden_states_123 = getitem_90.expand(1, 4, 8, 21, 64) + getitem_90 = None + key_24 = hidden_states_123.reshape(1, 32, 21, 64) + hidden_states_123 = None + getitem_91 = value_states_12[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_12 = None + hidden_states_124 = getitem_91.expand(1, 4, 8, 21, 64) + getitem_91 = None + value_24 = hidden_states_124.reshape(1, 32, 21, 64) + hidden_states_124 = None + attention_mask_13 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_12 = q_embed_12.contiguous() + q_embed_12 = None + key_25 = key_24.contiguous() + key_24 = None + value_25 = value_24.contiguous() + value_24 = None + attn_output_48 = torch._C._nn.scaled_dot_product_attention( + query_12, + key_25, + value_25, + attn_mask=attention_mask_13, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_12 = key_25 = value_25 = attention_mask_13 = None + transpose_52 = attn_output_48.transpose(1, 2) + attn_output_48 = None + attn_output_49 = transpose_52.contiguous() + transpose_52 = None + reshape_38 = attn_output_49.reshape(1, 21, -1) + attn_output_49 = None + attn_output_50 = reshape_38.contiguous() + reshape_38 = None + attn_output_51 = torch._C._nn.linear( + attn_output_50, + l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_50 = l_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_125 = hidden_states_119 + attn_output_51 + hidden_states_119 = attn_output_51 = None + hidden_states_126 = hidden_states_125.to(torch.float32) + pow_26 = hidden_states_126.pow(2) + variance_25 = pow_26.mean(-1, keepdim=True) + pow_26 = None + add_76 = variance_25 + 1e-05 + variance_25 = None + rsqrt_25 = torch.rsqrt(add_76) + add_76 = None + hidden_states_127 = hidden_states_126 * rsqrt_25 + hidden_states_126 = rsqrt_25 = None + to_55 = hidden_states_127.to(torch.float16) + hidden_states_127 = None + hidden_states_128 = ( + l_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ + * to_55 + ) + l_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_ = ( + to_55 + ) = None + linear_88 = torch._C._nn.linear( + hidden_states_128, + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_12 = torch.nn.functional.silu(linear_88, inplace=False) + linear_88 = None + linear_89 = torch._C._nn.linear( + hidden_states_128, + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_128 = l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_118 = silu_12 * linear_89 + silu_12 = linear_89 = None + down_proj_12 = torch._C._nn.linear( + mul_118, + l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_118 = l_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_129 = hidden_states_125 + down_proj_12 + hidden_states_125 = down_proj_12 = None + hidden_states_130 = hidden_states_129.to(torch.float32) + pow_27 = hidden_states_130.pow(2) + variance_26 = pow_27.mean(-1, keepdim=True) + pow_27 = None + add_78 = variance_26 + 1e-05 + variance_26 = None + rsqrt_26 = torch.rsqrt(add_78) + add_78 = None + hidden_states_131 = hidden_states_130 * rsqrt_26 + hidden_states_130 = rsqrt_26 = None + to_57 = hidden_states_131.to(torch.float16) + hidden_states_131 = None + hidden_states_132 = ( + l_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ + * to_57 + ) + l_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_ = ( + to_57 + ) = None + linear_91 = torch._C._nn.linear( + hidden_states_132, + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_39 = linear_91.view((1, 21, -1, 64)) + linear_91 = None + query_states_13 = view_39.transpose(1, 2) + view_39 = None + linear_92 = torch._C._nn.linear( + hidden_states_132, + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_40 = linear_92.view((1, 21, -1, 64)) + linear_92 = None + key_states_13 = view_40.transpose(1, 2) + view_40 = None + linear_93 = torch._C._nn.linear( + hidden_states_132, + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_132 = l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_41 = linear_93.view((1, 21, -1, 64)) + linear_93 = None + value_states_13 = view_41.transpose(1, 2) + view_41 = None + cos_16 = cos_2.unsqueeze(1) + sin_16 = sin_2.unsqueeze(1) + mul_121 = query_states_13 * cos_16 + x1_26 = query_states_13[(Ellipsis, slice(None, 32, None))] + x2_26 = query_states_13[(Ellipsis, slice(32, None, None))] + query_states_13 = None + neg_26 = -x2_26 + x2_26 = None + cat_27 = torch.cat((neg_26, x1_26), dim=-1) + neg_26 = x1_26 = None + mul_122 = cat_27 * sin_16 + cat_27 = None + q_embed_13 = mul_121 + mul_122 + mul_121 = mul_122 = None + mul_123 = key_states_13 * cos_16 + cos_16 = None + x1_27 = key_states_13[(Ellipsis, slice(None, 32, None))] + x2_27 = key_states_13[(Ellipsis, slice(32, None, None))] + key_states_13 = None + neg_27 = -x2_27 + x2_27 = None + cat_28 = torch.cat((neg_27, x1_27), dim=-1) + neg_27 = x1_27 = None + mul_124 = cat_28 * sin_16 + cat_28 = sin_16 = None + k_embed_13 = mul_123 + mul_124 + mul_123 = mul_124 = None + getitem_97 = k_embed_13[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_13 = None + hidden_states_133 = getitem_97.expand(1, 4, 8, 21, 64) + getitem_97 = None + key_26 = hidden_states_133.reshape(1, 32, 21, 64) + hidden_states_133 = None + getitem_98 = value_states_13[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_13 = None + hidden_states_134 = getitem_98.expand(1, 4, 8, 21, 64) + getitem_98 = None + value_26 = hidden_states_134.reshape(1, 32, 21, 64) + hidden_states_134 = None + attention_mask_14 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_13 = q_embed_13.contiguous() + q_embed_13 = None + key_27 = key_26.contiguous() + key_26 = None + value_27 = value_26.contiguous() + value_26 = None + attn_output_52 = torch._C._nn.scaled_dot_product_attention( + query_13, + key_27, + value_27, + attn_mask=attention_mask_14, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_13 = key_27 = value_27 = attention_mask_14 = None + transpose_56 = attn_output_52.transpose(1, 2) + attn_output_52 = None + attn_output_53 = transpose_56.contiguous() + transpose_56 = None + reshape_41 = attn_output_53.reshape(1, 21, -1) + attn_output_53 = None + attn_output_54 = reshape_41.contiguous() + reshape_41 = None + attn_output_55 = torch._C._nn.linear( + attn_output_54, + l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_54 = l_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_135 = hidden_states_129 + attn_output_55 + hidden_states_129 = attn_output_55 = None + hidden_states_136 = hidden_states_135.to(torch.float32) + pow_28 = hidden_states_136.pow(2) + variance_27 = pow_28.mean(-1, keepdim=True) + pow_28 = None + add_82 = variance_27 + 1e-05 + variance_27 = None + rsqrt_27 = torch.rsqrt(add_82) + add_82 = None + hidden_states_137 = hidden_states_136 * rsqrt_27 + hidden_states_136 = rsqrt_27 = None + to_59 = hidden_states_137.to(torch.float16) + hidden_states_137 = None + hidden_states_138 = ( + l_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ + * to_59 + ) + l_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_ = ( + to_59 + ) = None + linear_95 = torch._C._nn.linear( + hidden_states_138, + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_13 = torch.nn.functional.silu(linear_95, inplace=False) + linear_95 = None + linear_96 = torch._C._nn.linear( + hidden_states_138, + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_138 = l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_127 = silu_13 * linear_96 + silu_13 = linear_96 = None + down_proj_13 = torch._C._nn.linear( + mul_127, + l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_127 = l_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_139 = hidden_states_135 + down_proj_13 + hidden_states_135 = down_proj_13 = None + hidden_states_140 = hidden_states_139.to(torch.float32) + pow_29 = hidden_states_140.pow(2) + variance_28 = pow_29.mean(-1, keepdim=True) + pow_29 = None + add_84 = variance_28 + 1e-05 + variance_28 = None + rsqrt_28 = torch.rsqrt(add_84) + add_84 = None + hidden_states_141 = hidden_states_140 * rsqrt_28 + hidden_states_140 = rsqrt_28 = None + to_61 = hidden_states_141.to(torch.float16) + hidden_states_141 = None + hidden_states_142 = ( + l_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ + * to_61 + ) + l_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_ = ( + to_61 + ) = None + linear_98 = torch._C._nn.linear( + hidden_states_142, + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_42 = linear_98.view((1, 21, -1, 64)) + linear_98 = None + query_states_14 = view_42.transpose(1, 2) + view_42 = None + linear_99 = torch._C._nn.linear( + hidden_states_142, + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_43 = linear_99.view((1, 21, -1, 64)) + linear_99 = None + key_states_14 = view_43.transpose(1, 2) + view_43 = None + linear_100 = torch._C._nn.linear( + hidden_states_142, + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_142 = l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_44 = linear_100.view((1, 21, -1, 64)) + linear_100 = None + value_states_14 = view_44.transpose(1, 2) + view_44 = None + cos_17 = cos_2.unsqueeze(1) + sin_17 = sin_2.unsqueeze(1) + mul_130 = query_states_14 * cos_17 + x1_28 = query_states_14[(Ellipsis, slice(None, 32, None))] + x2_28 = query_states_14[(Ellipsis, slice(32, None, None))] + query_states_14 = None + neg_28 = -x2_28 + x2_28 = None + cat_29 = torch.cat((neg_28, x1_28), dim=-1) + neg_28 = x1_28 = None + mul_131 = cat_29 * sin_17 + cat_29 = None + q_embed_14 = mul_130 + mul_131 + mul_130 = mul_131 = None + mul_132 = key_states_14 * cos_17 + cos_17 = None + x1_29 = key_states_14[(Ellipsis, slice(None, 32, None))] + x2_29 = key_states_14[(Ellipsis, slice(32, None, None))] + key_states_14 = None + neg_29 = -x2_29 + x2_29 = None + cat_30 = torch.cat((neg_29, x1_29), dim=-1) + neg_29 = x1_29 = None + mul_133 = cat_30 * sin_17 + cat_30 = sin_17 = None + k_embed_14 = mul_132 + mul_133 + mul_132 = mul_133 = None + getitem_104 = k_embed_14[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_14 = None + hidden_states_143 = getitem_104.expand(1, 4, 8, 21, 64) + getitem_104 = None + key_28 = hidden_states_143.reshape(1, 32, 21, 64) + hidden_states_143 = None + getitem_105 = value_states_14[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_14 = None + hidden_states_144 = getitem_105.expand(1, 4, 8, 21, 64) + getitem_105 = None + value_28 = hidden_states_144.reshape(1, 32, 21, 64) + hidden_states_144 = None + attention_mask_15 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_14 = q_embed_14.contiguous() + q_embed_14 = None + key_29 = key_28.contiguous() + key_28 = None + value_29 = value_28.contiguous() + value_28 = None + attn_output_56 = torch._C._nn.scaled_dot_product_attention( + query_14, + key_29, + value_29, + attn_mask=attention_mask_15, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_14 = key_29 = value_29 = attention_mask_15 = None + transpose_60 = attn_output_56.transpose(1, 2) + attn_output_56 = None + attn_output_57 = transpose_60.contiguous() + transpose_60 = None + reshape_44 = attn_output_57.reshape(1, 21, -1) + attn_output_57 = None + attn_output_58 = reshape_44.contiguous() + reshape_44 = None + attn_output_59 = torch._C._nn.linear( + attn_output_58, + l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_58 = l_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_145 = hidden_states_139 + attn_output_59 + hidden_states_139 = attn_output_59 = None + hidden_states_146 = hidden_states_145.to(torch.float32) + pow_30 = hidden_states_146.pow(2) + variance_29 = pow_30.mean(-1, keepdim=True) + pow_30 = None + add_88 = variance_29 + 1e-05 + variance_29 = None + rsqrt_29 = torch.rsqrt(add_88) + add_88 = None + hidden_states_147 = hidden_states_146 * rsqrt_29 + hidden_states_146 = rsqrt_29 = None + to_63 = hidden_states_147.to(torch.float16) + hidden_states_147 = None + hidden_states_148 = ( + l_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ + * to_63 + ) + l_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_ = ( + to_63 + ) = None + linear_102 = torch._C._nn.linear( + hidden_states_148, + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_14 = torch.nn.functional.silu(linear_102, inplace=False) + linear_102 = None + linear_103 = torch._C._nn.linear( + hidden_states_148, + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_148 = l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_136 = silu_14 * linear_103 + silu_14 = linear_103 = None + down_proj_14 = torch._C._nn.linear( + mul_136, + l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_136 = l_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_149 = hidden_states_145 + down_proj_14 + hidden_states_145 = down_proj_14 = None + hidden_states_150 = hidden_states_149.to(torch.float32) + pow_31 = hidden_states_150.pow(2) + variance_30 = pow_31.mean(-1, keepdim=True) + pow_31 = None + add_90 = variance_30 + 1e-05 + variance_30 = None + rsqrt_30 = torch.rsqrt(add_90) + add_90 = None + hidden_states_151 = hidden_states_150 * rsqrt_30 + hidden_states_150 = rsqrt_30 = None + to_65 = hidden_states_151.to(torch.float16) + hidden_states_151 = None + hidden_states_152 = ( + l_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ + * to_65 + ) + l_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_ = ( + to_65 + ) = None + linear_105 = torch._C._nn.linear( + hidden_states_152, + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_45 = linear_105.view((1, 21, -1, 64)) + linear_105 = None + query_states_15 = view_45.transpose(1, 2) + view_45 = None + linear_106 = torch._C._nn.linear( + hidden_states_152, + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_46 = linear_106.view((1, 21, -1, 64)) + linear_106 = None + key_states_15 = view_46.transpose(1, 2) + view_46 = None + linear_107 = torch._C._nn.linear( + hidden_states_152, + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_152 = l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_47 = linear_107.view((1, 21, -1, 64)) + linear_107 = None + value_states_15 = view_47.transpose(1, 2) + view_47 = None + cos_18 = cos_2.unsqueeze(1) + sin_18 = sin_2.unsqueeze(1) + mul_139 = query_states_15 * cos_18 + x1_30 = query_states_15[(Ellipsis, slice(None, 32, None))] + x2_30 = query_states_15[(Ellipsis, slice(32, None, None))] + query_states_15 = None + neg_30 = -x2_30 + x2_30 = None + cat_31 = torch.cat((neg_30, x1_30), dim=-1) + neg_30 = x1_30 = None + mul_140 = cat_31 * sin_18 + cat_31 = None + q_embed_15 = mul_139 + mul_140 + mul_139 = mul_140 = None + mul_141 = key_states_15 * cos_18 + cos_18 = None + x1_31 = key_states_15[(Ellipsis, slice(None, 32, None))] + x2_31 = key_states_15[(Ellipsis, slice(32, None, None))] + key_states_15 = None + neg_31 = -x2_31 + x2_31 = None + cat_32 = torch.cat((neg_31, x1_31), dim=-1) + neg_31 = x1_31 = None + mul_142 = cat_32 * sin_18 + cat_32 = sin_18 = None + k_embed_15 = mul_141 + mul_142 + mul_141 = mul_142 = None + getitem_111 = k_embed_15[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_15 = None + hidden_states_153 = getitem_111.expand(1, 4, 8, 21, 64) + getitem_111 = None + key_30 = hidden_states_153.reshape(1, 32, 21, 64) + hidden_states_153 = None + getitem_112 = value_states_15[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_15 = None + hidden_states_154 = getitem_112.expand(1, 4, 8, 21, 64) + getitem_112 = None + value_30 = hidden_states_154.reshape(1, 32, 21, 64) + hidden_states_154 = None + attention_mask_16 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_15 = q_embed_15.contiguous() + q_embed_15 = None + key_31 = key_30.contiguous() + key_30 = None + value_31 = value_30.contiguous() + value_30 = None + attn_output_60 = torch._C._nn.scaled_dot_product_attention( + query_15, + key_31, + value_31, + attn_mask=attention_mask_16, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_15 = key_31 = value_31 = attention_mask_16 = None + transpose_64 = attn_output_60.transpose(1, 2) + attn_output_60 = None + attn_output_61 = transpose_64.contiguous() + transpose_64 = None + reshape_47 = attn_output_61.reshape(1, 21, -1) + attn_output_61 = None + attn_output_62 = reshape_47.contiguous() + reshape_47 = None + attn_output_63 = torch._C._nn.linear( + attn_output_62, + l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_62 = l_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_155 = hidden_states_149 + attn_output_63 + hidden_states_149 = attn_output_63 = None + hidden_states_156 = hidden_states_155.to(torch.float32) + pow_32 = hidden_states_156.pow(2) + variance_31 = pow_32.mean(-1, keepdim=True) + pow_32 = None + add_94 = variance_31 + 1e-05 + variance_31 = None + rsqrt_31 = torch.rsqrt(add_94) + add_94 = None + hidden_states_157 = hidden_states_156 * rsqrt_31 + hidden_states_156 = rsqrt_31 = None + to_67 = hidden_states_157.to(torch.float16) + hidden_states_157 = None + hidden_states_158 = ( + l_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ + * to_67 + ) + l_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_ = ( + to_67 + ) = None + linear_109 = torch._C._nn.linear( + hidden_states_158, + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_15 = torch.nn.functional.silu(linear_109, inplace=False) + linear_109 = None + linear_110 = torch._C._nn.linear( + hidden_states_158, + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_158 = l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_145 = silu_15 * linear_110 + silu_15 = linear_110 = None + down_proj_15 = torch._C._nn.linear( + mul_145, + l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_145 = l_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_159 = hidden_states_155 + down_proj_15 + hidden_states_155 = down_proj_15 = None + hidden_states_160 = hidden_states_159.to(torch.float32) + pow_33 = hidden_states_160.pow(2) + variance_32 = pow_33.mean(-1, keepdim=True) + pow_33 = None + add_96 = variance_32 + 1e-05 + variance_32 = None + rsqrt_32 = torch.rsqrt(add_96) + add_96 = None + hidden_states_161 = hidden_states_160 * rsqrt_32 + hidden_states_160 = rsqrt_32 = None + to_69 = hidden_states_161.to(torch.float16) + hidden_states_161 = None + hidden_states_162 = ( + l_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ + * to_69 + ) + l_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_ = ( + to_69 + ) = None + linear_112 = torch._C._nn.linear( + hidden_states_162, + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_48 = linear_112.view((1, 21, -1, 64)) + linear_112 = None + query_states_16 = view_48.transpose(1, 2) + view_48 = None + linear_113 = torch._C._nn.linear( + hidden_states_162, + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_49 = linear_113.view((1, 21, -1, 64)) + linear_113 = None + key_states_16 = view_49.transpose(1, 2) + view_49 = None + linear_114 = torch._C._nn.linear( + hidden_states_162, + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_162 = l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_50 = linear_114.view((1, 21, -1, 64)) + linear_114 = None + value_states_16 = view_50.transpose(1, 2) + view_50 = None + cos_19 = cos_2.unsqueeze(1) + sin_19 = sin_2.unsqueeze(1) + mul_148 = query_states_16 * cos_19 + x1_32 = query_states_16[(Ellipsis, slice(None, 32, None))] + x2_32 = query_states_16[(Ellipsis, slice(32, None, None))] + query_states_16 = None + neg_32 = -x2_32 + x2_32 = None + cat_33 = torch.cat((neg_32, x1_32), dim=-1) + neg_32 = x1_32 = None + mul_149 = cat_33 * sin_19 + cat_33 = None + q_embed_16 = mul_148 + mul_149 + mul_148 = mul_149 = None + mul_150 = key_states_16 * cos_19 + cos_19 = None + x1_33 = key_states_16[(Ellipsis, slice(None, 32, None))] + x2_33 = key_states_16[(Ellipsis, slice(32, None, None))] + key_states_16 = None + neg_33 = -x2_33 + x2_33 = None + cat_34 = torch.cat((neg_33, x1_33), dim=-1) + neg_33 = x1_33 = None + mul_151 = cat_34 * sin_19 + cat_34 = sin_19 = None + k_embed_16 = mul_150 + mul_151 + mul_150 = mul_151 = None + getitem_118 = k_embed_16[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_16 = None + hidden_states_163 = getitem_118.expand(1, 4, 8, 21, 64) + getitem_118 = None + key_32 = hidden_states_163.reshape(1, 32, 21, 64) + hidden_states_163 = None + getitem_119 = value_states_16[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_16 = None + hidden_states_164 = getitem_119.expand(1, 4, 8, 21, 64) + getitem_119 = None + value_32 = hidden_states_164.reshape(1, 32, 21, 64) + hidden_states_164 = None + attention_mask_17 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_16 = q_embed_16.contiguous() + q_embed_16 = None + key_33 = key_32.contiguous() + key_32 = None + value_33 = value_32.contiguous() + value_32 = None + attn_output_64 = torch._C._nn.scaled_dot_product_attention( + query_16, + key_33, + value_33, + attn_mask=attention_mask_17, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_16 = key_33 = value_33 = attention_mask_17 = None + transpose_68 = attn_output_64.transpose(1, 2) + attn_output_64 = None + attn_output_65 = transpose_68.contiguous() + transpose_68 = None + reshape_50 = attn_output_65.reshape(1, 21, -1) + attn_output_65 = None + attn_output_66 = reshape_50.contiguous() + reshape_50 = None + attn_output_67 = torch._C._nn.linear( + attn_output_66, + l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_66 = l_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_165 = hidden_states_159 + attn_output_67 + hidden_states_159 = attn_output_67 = None + hidden_states_166 = hidden_states_165.to(torch.float32) + pow_34 = hidden_states_166.pow(2) + variance_33 = pow_34.mean(-1, keepdim=True) + pow_34 = None + add_100 = variance_33 + 1e-05 + variance_33 = None + rsqrt_33 = torch.rsqrt(add_100) + add_100 = None + hidden_states_167 = hidden_states_166 * rsqrt_33 + hidden_states_166 = rsqrt_33 = None + to_71 = hidden_states_167.to(torch.float16) + hidden_states_167 = None + hidden_states_168 = ( + l_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ + * to_71 + ) + l_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_ = ( + to_71 + ) = None + linear_116 = torch._C._nn.linear( + hidden_states_168, + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_16 = torch.nn.functional.silu(linear_116, inplace=False) + linear_116 = None + linear_117 = torch._C._nn.linear( + hidden_states_168, + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_168 = l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_154 = silu_16 * linear_117 + silu_16 = linear_117 = None + down_proj_16 = torch._C._nn.linear( + mul_154, + l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_154 = l_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_169 = hidden_states_165 + down_proj_16 + hidden_states_165 = down_proj_16 = None + hidden_states_170 = hidden_states_169.to(torch.float32) + pow_35 = hidden_states_170.pow(2) + variance_34 = pow_35.mean(-1, keepdim=True) + pow_35 = None + add_102 = variance_34 + 1e-05 + variance_34 = None + rsqrt_34 = torch.rsqrt(add_102) + add_102 = None + hidden_states_171 = hidden_states_170 * rsqrt_34 + hidden_states_170 = rsqrt_34 = None + to_73 = hidden_states_171.to(torch.float16) + hidden_states_171 = None + hidden_states_172 = ( + l_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ + * to_73 + ) + l_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_ = ( + to_73 + ) = None + linear_119 = torch._C._nn.linear( + hidden_states_172, + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_51 = linear_119.view((1, 21, -1, 64)) + linear_119 = None + query_states_17 = view_51.transpose(1, 2) + view_51 = None + linear_120 = torch._C._nn.linear( + hidden_states_172, + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_52 = linear_120.view((1, 21, -1, 64)) + linear_120 = None + key_states_17 = view_52.transpose(1, 2) + view_52 = None + linear_121 = torch._C._nn.linear( + hidden_states_172, + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_172 = l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_53 = linear_121.view((1, 21, -1, 64)) + linear_121 = None + value_states_17 = view_53.transpose(1, 2) + view_53 = None + cos_20 = cos_2.unsqueeze(1) + sin_20 = sin_2.unsqueeze(1) + mul_157 = query_states_17 * cos_20 + x1_34 = query_states_17[(Ellipsis, slice(None, 32, None))] + x2_34 = query_states_17[(Ellipsis, slice(32, None, None))] + query_states_17 = None + neg_34 = -x2_34 + x2_34 = None + cat_35 = torch.cat((neg_34, x1_34), dim=-1) + neg_34 = x1_34 = None + mul_158 = cat_35 * sin_20 + cat_35 = None + q_embed_17 = mul_157 + mul_158 + mul_157 = mul_158 = None + mul_159 = key_states_17 * cos_20 + cos_20 = None + x1_35 = key_states_17[(Ellipsis, slice(None, 32, None))] + x2_35 = key_states_17[(Ellipsis, slice(32, None, None))] + key_states_17 = None + neg_35 = -x2_35 + x2_35 = None + cat_36 = torch.cat((neg_35, x1_35), dim=-1) + neg_35 = x1_35 = None + mul_160 = cat_36 * sin_20 + cat_36 = sin_20 = None + k_embed_17 = mul_159 + mul_160 + mul_159 = mul_160 = None + getitem_125 = k_embed_17[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_17 = None + hidden_states_173 = getitem_125.expand(1, 4, 8, 21, 64) + getitem_125 = None + key_34 = hidden_states_173.reshape(1, 32, 21, 64) + hidden_states_173 = None + getitem_126 = value_states_17[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_17 = None + hidden_states_174 = getitem_126.expand(1, 4, 8, 21, 64) + getitem_126 = None + value_34 = hidden_states_174.reshape(1, 32, 21, 64) + hidden_states_174 = None + attention_mask_18 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_17 = q_embed_17.contiguous() + q_embed_17 = None + key_35 = key_34.contiguous() + key_34 = None + value_35 = value_34.contiguous() + value_34 = None + attn_output_68 = torch._C._nn.scaled_dot_product_attention( + query_17, + key_35, + value_35, + attn_mask=attention_mask_18, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_17 = key_35 = value_35 = attention_mask_18 = None + transpose_72 = attn_output_68.transpose(1, 2) + attn_output_68 = None + attn_output_69 = transpose_72.contiguous() + transpose_72 = None + reshape_53 = attn_output_69.reshape(1, 21, -1) + attn_output_69 = None + attn_output_70 = reshape_53.contiguous() + reshape_53 = None + attn_output_71 = torch._C._nn.linear( + attn_output_70, + l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_70 = l_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_175 = hidden_states_169 + attn_output_71 + hidden_states_169 = attn_output_71 = None + hidden_states_176 = hidden_states_175.to(torch.float32) + pow_36 = hidden_states_176.pow(2) + variance_35 = pow_36.mean(-1, keepdim=True) + pow_36 = None + add_106 = variance_35 + 1e-05 + variance_35 = None + rsqrt_35 = torch.rsqrt(add_106) + add_106 = None + hidden_states_177 = hidden_states_176 * rsqrt_35 + hidden_states_176 = rsqrt_35 = None + to_75 = hidden_states_177.to(torch.float16) + hidden_states_177 = None + hidden_states_178 = ( + l_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ + * to_75 + ) + l_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_ = ( + to_75 + ) = None + linear_123 = torch._C._nn.linear( + hidden_states_178, + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_17 = torch.nn.functional.silu(linear_123, inplace=False) + linear_123 = None + linear_124 = torch._C._nn.linear( + hidden_states_178, + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_178 = l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_163 = silu_17 * linear_124 + silu_17 = linear_124 = None + down_proj_17 = torch._C._nn.linear( + mul_163, + l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_163 = l_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_179 = hidden_states_175 + down_proj_17 + hidden_states_175 = down_proj_17 = None + hidden_states_180 = hidden_states_179.to(torch.float32) + pow_37 = hidden_states_180.pow(2) + variance_36 = pow_37.mean(-1, keepdim=True) + pow_37 = None + add_108 = variance_36 + 1e-05 + variance_36 = None + rsqrt_36 = torch.rsqrt(add_108) + add_108 = None + hidden_states_181 = hidden_states_180 * rsqrt_36 + hidden_states_180 = rsqrt_36 = None + to_77 = hidden_states_181.to(torch.float16) + hidden_states_181 = None + hidden_states_182 = ( + l_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ + * to_77 + ) + l_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_ = ( + to_77 + ) = None + linear_126 = torch._C._nn.linear( + hidden_states_182, + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_54 = linear_126.view((1, 21, -1, 64)) + linear_126 = None + query_states_18 = view_54.transpose(1, 2) + view_54 = None + linear_127 = torch._C._nn.linear( + hidden_states_182, + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_55 = linear_127.view((1, 21, -1, 64)) + linear_127 = None + key_states_18 = view_55.transpose(1, 2) + view_55 = None + linear_128 = torch._C._nn.linear( + hidden_states_182, + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_182 = l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_56 = linear_128.view((1, 21, -1, 64)) + linear_128 = None + value_states_18 = view_56.transpose(1, 2) + view_56 = None + cos_21 = cos_2.unsqueeze(1) + sin_21 = sin_2.unsqueeze(1) + mul_166 = query_states_18 * cos_21 + x1_36 = query_states_18[(Ellipsis, slice(None, 32, None))] + x2_36 = query_states_18[(Ellipsis, slice(32, None, None))] + query_states_18 = None + neg_36 = -x2_36 + x2_36 = None + cat_37 = torch.cat((neg_36, x1_36), dim=-1) + neg_36 = x1_36 = None + mul_167 = cat_37 * sin_21 + cat_37 = None + q_embed_18 = mul_166 + mul_167 + mul_166 = mul_167 = None + mul_168 = key_states_18 * cos_21 + cos_21 = None + x1_37 = key_states_18[(Ellipsis, slice(None, 32, None))] + x2_37 = key_states_18[(Ellipsis, slice(32, None, None))] + key_states_18 = None + neg_37 = -x2_37 + x2_37 = None + cat_38 = torch.cat((neg_37, x1_37), dim=-1) + neg_37 = x1_37 = None + mul_169 = cat_38 * sin_21 + cat_38 = sin_21 = None + k_embed_18 = mul_168 + mul_169 + mul_168 = mul_169 = None + getitem_132 = k_embed_18[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_18 = None + hidden_states_183 = getitem_132.expand(1, 4, 8, 21, 64) + getitem_132 = None + key_36 = hidden_states_183.reshape(1, 32, 21, 64) + hidden_states_183 = None + getitem_133 = value_states_18[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_18 = None + hidden_states_184 = getitem_133.expand(1, 4, 8, 21, 64) + getitem_133 = None + value_36 = hidden_states_184.reshape(1, 32, 21, 64) + hidden_states_184 = None + attention_mask_19 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_18 = q_embed_18.contiguous() + q_embed_18 = None + key_37 = key_36.contiguous() + key_36 = None + value_37 = value_36.contiguous() + value_36 = None + attn_output_72 = torch._C._nn.scaled_dot_product_attention( + query_18, + key_37, + value_37, + attn_mask=attention_mask_19, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_18 = key_37 = value_37 = attention_mask_19 = None + transpose_76 = attn_output_72.transpose(1, 2) + attn_output_72 = None + attn_output_73 = transpose_76.contiguous() + transpose_76 = None + reshape_56 = attn_output_73.reshape(1, 21, -1) + attn_output_73 = None + attn_output_74 = reshape_56.contiguous() + reshape_56 = None + attn_output_75 = torch._C._nn.linear( + attn_output_74, + l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_74 = l_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_185 = hidden_states_179 + attn_output_75 + hidden_states_179 = attn_output_75 = None + hidden_states_186 = hidden_states_185.to(torch.float32) + pow_38 = hidden_states_186.pow(2) + variance_37 = pow_38.mean(-1, keepdim=True) + pow_38 = None + add_112 = variance_37 + 1e-05 + variance_37 = None + rsqrt_37 = torch.rsqrt(add_112) + add_112 = None + hidden_states_187 = hidden_states_186 * rsqrt_37 + hidden_states_186 = rsqrt_37 = None + to_79 = hidden_states_187.to(torch.float16) + hidden_states_187 = None + hidden_states_188 = ( + l_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ + * to_79 + ) + l_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_ = ( + to_79 + ) = None + linear_130 = torch._C._nn.linear( + hidden_states_188, + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_18 = torch.nn.functional.silu(linear_130, inplace=False) + linear_130 = None + linear_131 = torch._C._nn.linear( + hidden_states_188, + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_188 = l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_172 = silu_18 * linear_131 + silu_18 = linear_131 = None + down_proj_18 = torch._C._nn.linear( + mul_172, + l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_172 = l_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_189 = hidden_states_185 + down_proj_18 + hidden_states_185 = down_proj_18 = None + hidden_states_190 = hidden_states_189.to(torch.float32) + pow_39 = hidden_states_190.pow(2) + variance_38 = pow_39.mean(-1, keepdim=True) + pow_39 = None + add_114 = variance_38 + 1e-05 + variance_38 = None + rsqrt_38 = torch.rsqrt(add_114) + add_114 = None + hidden_states_191 = hidden_states_190 * rsqrt_38 + hidden_states_190 = rsqrt_38 = None + to_81 = hidden_states_191.to(torch.float16) + hidden_states_191 = None + hidden_states_192 = ( + l_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ + * to_81 + ) + l_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_ = ( + to_81 + ) = None + linear_133 = torch._C._nn.linear( + hidden_states_192, + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_57 = linear_133.view((1, 21, -1, 64)) + linear_133 = None + query_states_19 = view_57.transpose(1, 2) + view_57 = None + linear_134 = torch._C._nn.linear( + hidden_states_192, + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_58 = linear_134.view((1, 21, -1, 64)) + linear_134 = None + key_states_19 = view_58.transpose(1, 2) + view_58 = None + linear_135 = torch._C._nn.linear( + hidden_states_192, + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_192 = l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_59 = linear_135.view((1, 21, -1, 64)) + linear_135 = None + value_states_19 = view_59.transpose(1, 2) + view_59 = None + cos_22 = cos_2.unsqueeze(1) + sin_22 = sin_2.unsqueeze(1) + mul_175 = query_states_19 * cos_22 + x1_38 = query_states_19[(Ellipsis, slice(None, 32, None))] + x2_38 = query_states_19[(Ellipsis, slice(32, None, None))] + query_states_19 = None + neg_38 = -x2_38 + x2_38 = None + cat_39 = torch.cat((neg_38, x1_38), dim=-1) + neg_38 = x1_38 = None + mul_176 = cat_39 * sin_22 + cat_39 = None + q_embed_19 = mul_175 + mul_176 + mul_175 = mul_176 = None + mul_177 = key_states_19 * cos_22 + cos_22 = None + x1_39 = key_states_19[(Ellipsis, slice(None, 32, None))] + x2_39 = key_states_19[(Ellipsis, slice(32, None, None))] + key_states_19 = None + neg_39 = -x2_39 + x2_39 = None + cat_40 = torch.cat((neg_39, x1_39), dim=-1) + neg_39 = x1_39 = None + mul_178 = cat_40 * sin_22 + cat_40 = sin_22 = None + k_embed_19 = mul_177 + mul_178 + mul_177 = mul_178 = None + getitem_139 = k_embed_19[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_19 = None + hidden_states_193 = getitem_139.expand(1, 4, 8, 21, 64) + getitem_139 = None + key_38 = hidden_states_193.reshape(1, 32, 21, 64) + hidden_states_193 = None + getitem_140 = value_states_19[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_19 = None + hidden_states_194 = getitem_140.expand(1, 4, 8, 21, 64) + getitem_140 = None + value_38 = hidden_states_194.reshape(1, 32, 21, 64) + hidden_states_194 = None + attention_mask_20 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_19 = q_embed_19.contiguous() + q_embed_19 = None + key_39 = key_38.contiguous() + key_38 = None + value_39 = value_38.contiguous() + value_38 = None + attn_output_76 = torch._C._nn.scaled_dot_product_attention( + query_19, + key_39, + value_39, + attn_mask=attention_mask_20, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_19 = key_39 = value_39 = attention_mask_20 = None + transpose_80 = attn_output_76.transpose(1, 2) + attn_output_76 = None + attn_output_77 = transpose_80.contiguous() + transpose_80 = None + reshape_59 = attn_output_77.reshape(1, 21, -1) + attn_output_77 = None + attn_output_78 = reshape_59.contiguous() + reshape_59 = None + attn_output_79 = torch._C._nn.linear( + attn_output_78, + l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_78 = l_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_195 = hidden_states_189 + attn_output_79 + hidden_states_189 = attn_output_79 = None + hidden_states_196 = hidden_states_195.to(torch.float32) + pow_40 = hidden_states_196.pow(2) + variance_39 = pow_40.mean(-1, keepdim=True) + pow_40 = None + add_118 = variance_39 + 1e-05 + variance_39 = None + rsqrt_39 = torch.rsqrt(add_118) + add_118 = None + hidden_states_197 = hidden_states_196 * rsqrt_39 + hidden_states_196 = rsqrt_39 = None + to_83 = hidden_states_197.to(torch.float16) + hidden_states_197 = None + hidden_states_198 = ( + l_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ + * to_83 + ) + l_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_ = ( + to_83 + ) = None + linear_137 = torch._C._nn.linear( + hidden_states_198, + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_19 = torch.nn.functional.silu(linear_137, inplace=False) + linear_137 = None + linear_138 = torch._C._nn.linear( + hidden_states_198, + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_198 = l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_181 = silu_19 * linear_138 + silu_19 = linear_138 = None + down_proj_19 = torch._C._nn.linear( + mul_181, + l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_181 = l_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_199 = hidden_states_195 + down_proj_19 + hidden_states_195 = down_proj_19 = None + hidden_states_200 = hidden_states_199.to(torch.float32) + pow_41 = hidden_states_200.pow(2) + variance_40 = pow_41.mean(-1, keepdim=True) + pow_41 = None + add_120 = variance_40 + 1e-05 + variance_40 = None + rsqrt_40 = torch.rsqrt(add_120) + add_120 = None + hidden_states_201 = hidden_states_200 * rsqrt_40 + hidden_states_200 = rsqrt_40 = None + to_85 = hidden_states_201.to(torch.float16) + hidden_states_201 = None + hidden_states_202 = ( + l_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ + * to_85 + ) + l_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_ = ( + to_85 + ) = None + linear_140 = torch._C._nn.linear( + hidden_states_202, + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_60 = linear_140.view((1, 21, -1, 64)) + linear_140 = None + query_states_20 = view_60.transpose(1, 2) + view_60 = None + linear_141 = torch._C._nn.linear( + hidden_states_202, + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_61 = linear_141.view((1, 21, -1, 64)) + linear_141 = None + key_states_20 = view_61.transpose(1, 2) + view_61 = None + linear_142 = torch._C._nn.linear( + hidden_states_202, + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_202 = l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_62 = linear_142.view((1, 21, -1, 64)) + linear_142 = None + value_states_20 = view_62.transpose(1, 2) + view_62 = None + cos_23 = cos_2.unsqueeze(1) + sin_23 = sin_2.unsqueeze(1) + mul_184 = query_states_20 * cos_23 + x1_40 = query_states_20[(Ellipsis, slice(None, 32, None))] + x2_40 = query_states_20[(Ellipsis, slice(32, None, None))] + query_states_20 = None + neg_40 = -x2_40 + x2_40 = None + cat_41 = torch.cat((neg_40, x1_40), dim=-1) + neg_40 = x1_40 = None + mul_185 = cat_41 * sin_23 + cat_41 = None + q_embed_20 = mul_184 + mul_185 + mul_184 = mul_185 = None + mul_186 = key_states_20 * cos_23 + cos_23 = None + x1_41 = key_states_20[(Ellipsis, slice(None, 32, None))] + x2_41 = key_states_20[(Ellipsis, slice(32, None, None))] + key_states_20 = None + neg_41 = -x2_41 + x2_41 = None + cat_42 = torch.cat((neg_41, x1_41), dim=-1) + neg_41 = x1_41 = None + mul_187 = cat_42 * sin_23 + cat_42 = sin_23 = None + k_embed_20 = mul_186 + mul_187 + mul_186 = mul_187 = None + getitem_146 = k_embed_20[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_20 = None + hidden_states_203 = getitem_146.expand(1, 4, 8, 21, 64) + getitem_146 = None + key_40 = hidden_states_203.reshape(1, 32, 21, 64) + hidden_states_203 = None + getitem_147 = value_states_20[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_20 = None + hidden_states_204 = getitem_147.expand(1, 4, 8, 21, 64) + getitem_147 = None + value_40 = hidden_states_204.reshape(1, 32, 21, 64) + hidden_states_204 = None + attention_mask_21 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + query_20 = q_embed_20.contiguous() + q_embed_20 = None + key_41 = key_40.contiguous() + key_40 = None + value_41 = value_40.contiguous() + value_40 = None + attn_output_80 = torch._C._nn.scaled_dot_product_attention( + query_20, + key_41, + value_41, + attn_mask=attention_mask_21, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_20 = key_41 = value_41 = attention_mask_21 = None + transpose_84 = attn_output_80.transpose(1, 2) + attn_output_80 = None + attn_output_81 = transpose_84.contiguous() + transpose_84 = None + reshape_62 = attn_output_81.reshape(1, 21, -1) + attn_output_81 = None + attn_output_82 = reshape_62.contiguous() + reshape_62 = None + attn_output_83 = torch._C._nn.linear( + attn_output_82, + l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_82 = l_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_205 = hidden_states_199 + attn_output_83 + hidden_states_199 = attn_output_83 = None + hidden_states_206 = hidden_states_205.to(torch.float32) + pow_42 = hidden_states_206.pow(2) + variance_41 = pow_42.mean(-1, keepdim=True) + pow_42 = None + add_124 = variance_41 + 1e-05 + variance_41 = None + rsqrt_41 = torch.rsqrt(add_124) + add_124 = None + hidden_states_207 = hidden_states_206 * rsqrt_41 + hidden_states_206 = rsqrt_41 = None + to_87 = hidden_states_207.to(torch.float16) + hidden_states_207 = None + hidden_states_208 = ( + l_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ + * to_87 + ) + l_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_ = ( + to_87 + ) = None + linear_144 = torch._C._nn.linear( + hidden_states_208, + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_20 = torch.nn.functional.silu(linear_144, inplace=False) + linear_144 = None + linear_145 = torch._C._nn.linear( + hidden_states_208, + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_208 = l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_190 = silu_20 * linear_145 + silu_20 = linear_145 = None + down_proj_20 = torch._C._nn.linear( + mul_190, + l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_190 = l_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_209 = hidden_states_205 + down_proj_20 + hidden_states_205 = down_proj_20 = None + hidden_states_210 = hidden_states_209.to(torch.float32) + pow_43 = hidden_states_210.pow(2) + variance_42 = pow_43.mean(-1, keepdim=True) + pow_43 = None + add_126 = variance_42 + 1e-05 + variance_42 = None + rsqrt_42 = torch.rsqrt(add_126) + add_126 = None + hidden_states_211 = hidden_states_210 * rsqrt_42 + hidden_states_210 = rsqrt_42 = None + to_89 = hidden_states_211.to(torch.float16) + hidden_states_211 = None + hidden_states_212 = ( + l_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ + * to_89 + ) + l_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_ = ( + to_89 + ) = None + linear_147 = torch._C._nn.linear( + hidden_states_212, + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_ = ( + None + ) + view_63 = linear_147.view((1, 21, -1, 64)) + linear_147 = None + query_states_21 = view_63.transpose(1, 2) + view_63 = None + linear_148 = torch._C._nn.linear( + hidden_states_212, + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_ = ( + None + ) + view_64 = linear_148.view((1, 21, -1, 64)) + linear_148 = None + key_states_21 = view_64.transpose(1, 2) + view_64 = None + linear_149 = torch._C._nn.linear( + hidden_states_212, + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_, + None, + ) + hidden_states_212 = l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_ = (None) + view_65 = linear_149.view((1, 21, -1, 64)) + linear_149 = None + value_states_21 = view_65.transpose(1, 2) + view_65 = None + cos_24 = cos_2.unsqueeze(1) + cos_2 = None + sin_24 = sin_2.unsqueeze(1) + sin_2 = None + mul_193 = query_states_21 * cos_24 + x1_42 = query_states_21[(Ellipsis, slice(None, 32, None))] + x2_42 = query_states_21[(Ellipsis, slice(32, None, None))] + query_states_21 = None + neg_42 = -x2_42 + x2_42 = None + cat_43 = torch.cat((neg_42, x1_42), dim=-1) + neg_42 = x1_42 = None + mul_194 = cat_43 * sin_24 + cat_43 = None + q_embed_21 = mul_193 + mul_194 + mul_193 = mul_194 = None + mul_195 = key_states_21 * cos_24 + cos_24 = None + x1_43 = key_states_21[(Ellipsis, slice(None, 32, None))] + x2_43 = key_states_21[(Ellipsis, slice(32, None, None))] + key_states_21 = None + neg_43 = -x2_43 + x2_43 = None + cat_44 = torch.cat((neg_43, x1_43), dim=-1) + neg_43 = x1_43 = None + mul_196 = cat_44 * sin_24 + cat_44 = sin_24 = None + k_embed_21 = mul_195 + mul_196 + mul_195 = mul_196 = None + getitem_153 = k_embed_21[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + k_embed_21 = None + hidden_states_213 = getitem_153.expand(1, 4, 8, 21, 64) + getitem_153 = None + key_42 = hidden_states_213.reshape(1, 32, 21, 64) + hidden_states_213 = None + getitem_154 = value_states_21[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_states_21 = None + hidden_states_214 = getitem_154.expand(1, 4, 8, 21, 64) + getitem_154 = None + value_42 = hidden_states_214.reshape(1, 32, 21, 64) + hidden_states_214 = None + attention_mask_22 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 21, None), + ) + ] + causal_mask = None + query_21 = q_embed_21.contiguous() + q_embed_21 = None + key_43 = key_42.contiguous() + key_42 = None + value_43 = value_42.contiguous() + value_42 = None + attn_output_84 = torch._C._nn.scaled_dot_product_attention( + query_21, + key_43, + value_43, + attn_mask=attention_mask_22, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_21 = key_43 = value_43 = attention_mask_22 = None + transpose_88 = attn_output_84.transpose(1, 2) + attn_output_84 = None + attn_output_85 = transpose_88.contiguous() + transpose_88 = None + reshape_65 = attn_output_85.reshape(1, 21, -1) + attn_output_85 = None + attn_output_86 = reshape_65.contiguous() + reshape_65 = None + attn_output_87 = torch._C._nn.linear( + attn_output_86, + l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_, + None, + ) + attn_output_86 = l_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_ = (None) + hidden_states_215 = hidden_states_209 + attn_output_87 + hidden_states_209 = attn_output_87 = None + hidden_states_216 = hidden_states_215.to(torch.float32) + pow_44 = hidden_states_216.pow(2) + variance_43 = pow_44.mean(-1, keepdim=True) + pow_44 = None + add_130 = variance_43 + 1e-05 + variance_43 = None + rsqrt_43 = torch.rsqrt(add_130) + add_130 = None + hidden_states_217 = hidden_states_216 * rsqrt_43 + hidden_states_216 = rsqrt_43 = None + to_91 = hidden_states_217.to(torch.float16) + hidden_states_217 = None + hidden_states_218 = ( + l_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ + * to_91 + ) + l_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_ = ( + to_91 + ) = None + linear_151 = torch._C._nn.linear( + hidden_states_218, + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_, + None, + ) + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_ = ( + None + ) + silu_21 = torch.nn.functional.silu(linear_151, inplace=False) + linear_151 = None + linear_152 = torch._C._nn.linear( + hidden_states_218, + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_, + None, + ) + hidden_states_218 = l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_ = (None) + mul_199 = silu_21 * linear_152 + silu_21 = linear_152 = None + down_proj_21 = torch._C._nn.linear( + mul_199, + l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_, + None, + ) + mul_199 = l_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_ = (None) + hidden_states_219 = hidden_states_215 + down_proj_21 + hidden_states_215 = down_proj_21 = None + hidden_states_220 = hidden_states_219.to(torch.float32) + hidden_states_219 = None + pow_45 = hidden_states_220.pow(2) + variance_44 = pow_45.mean(-1, keepdim=True) + pow_45 = None + add_132 = variance_44 + 1e-05 + variance_44 = None + rsqrt_44 = torch.rsqrt(add_132) + add_132 = None + hidden_states_221 = hidden_states_220 * rsqrt_44 + hidden_states_220 = rsqrt_44 = None + to_93 = hidden_states_221.to(torch.float16) + hidden_states_221 = None + hidden_states_222 = l_self_modules_model_modules_norm_parameters_weight_ * to_93 + l_self_modules_model_modules_norm_parameters_weight_ = to_93 = None + getitem_156 = hidden_states_222[ + (slice(None, None, None), slice(0, None, None), slice(None, None, None)) + ] + hidden_states_222 = None + logits = torch._C._nn.linear( + getitem_156, l_self_modules_lm_head_parameters_weight_, None + ) + getitem_156 = l_self_modules_lm_head_parameters_weight_ = None + return (logits,) diff --git a/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/weight_meta.py b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/weight_meta.py new file mode 100644 index 0000000000..c216b48e4a --- /dev/null +++ b/samples/transformers-auto-model/TinyLlama-1.1B-step-50K-105b/weight_meta.py @@ -0,0 +1,2060 @@ +class Program_weight_tensor_meta_L_kwargs_input_ids_: + name = "L_kwargs_input_ids_" + shape = [1, 21] + dtype = "torch.int64" + device = "cpu" + mean = None + std = None + data = [ + 1, + 15043, + 29892, + 590, + 1024, + 338, + 7991, + 29889, + 306, + 626, + 6509, + 1048, + 2919, + 4086, + 4733, + 322, + 1009, + 6956, + 1973, + 29889, + 29871, + ] + + +class Program_weight_tensor_meta_L_self_modules_model_modules_embed_tokens_parameters_weight_: + name = "L_self_modules_model_modules_embed_tokens_parameters_weight_" + shape = [32000, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_kwargs_attention_mask_: + name = "L_kwargs_attention_mask_" + shape = [1, 21] + dtype = "torch.int64" + device = "cpu" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_model_modules_rotary_emb_buffers_inv_freq_: + name = "L_self_modules_model_modules_rotary_emb_buffers_inv_freq_" + shape = [32] + dtype = "torch.float32" + device = "cpu" + mean = 0.125 + std = 0.240 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_0_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_1_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_2_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_3_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_4_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_5_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_6_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_7_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_8_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_9_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_10_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_11_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_12_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_13_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_14_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_15_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_16_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_17_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_18_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_19_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_20_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_input_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [256, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_self_attn_modules_o_proj_parameters_weight_" + shape = [2048, 2048] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_post_attention_layernorm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_gate_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_up_proj_parameters_weight_" + shape = [5632, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_: + name = "L_self_modules_model_modules_layers_modules_21_modules_mlp_modules_down_proj_parameters_weight_" + shape = [2048, 5632] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_model_modules_norm_parameters_weight_: + name = "L_self_modules_model_modules_norm_parameters_weight_" + shape = [2048] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_lm_head_parameters_weight_: + name = "L_self_modules_lm_head_parameters_weight_" + shape = [32000, 2048] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None diff --git a/samples/transformers-auto-model/phi-2/graph_hash.txt b/samples/transformers-auto-model/phi-2/graph_hash.txt new file mode 100644 index 0000000000..149a25140a --- /dev/null +++ b/samples/transformers-auto-model/phi-2/graph_hash.txt @@ -0,0 +1 @@ +8f6ea170d6349342ca4d6948527db486e5ea9acf3824a2153397c644f0802dac \ No newline at end of file diff --git a/samples/transformers-auto-model/phi-2/graph_net.json b/samples/transformers-auto-model/phi-2/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/phi-2/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/phi-2/input_meta.py b/samples/transformers-auto-model/phi-2/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/phi-2/input_tensor_constraints.py b/samples/transformers-auto-model/phi-2/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/phi-2/model.py b/samples/transformers-auto-model/phi-2/model.py new file mode 100644 index 0000000000..abf3bc968e --- /dev/null +++ b/samples/transformers-auto-model/phi-2/model.py @@ -0,0 +1,40 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_self_modules_ln_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_ln_parameters_bias_: torch.nn.parameter.Parameter, + L_hidden_states_: torch.Tensor, + L_self_modules_linear_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_linear_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_self_modules_ln_parameters_weight_ = L_self_modules_ln_parameters_weight_ + l_self_modules_ln_parameters_bias_ = L_self_modules_ln_parameters_bias_ + l_hidden_states_ = L_hidden_states_ + l_self_modules_linear_parameters_weight_ = ( + L_self_modules_linear_parameters_weight_ + ) + l_self_modules_linear_parameters_bias_ = L_self_modules_linear_parameters_bias_ + hidden_states = torch.nn.functional.layer_norm( + l_hidden_states_, + (2560,), + l_self_modules_ln_parameters_weight_, + l_self_modules_ln_parameters_bias_, + 1e-05, + ) + l_hidden_states_ = ( + l_self_modules_ln_parameters_weight_ + ) = l_self_modules_ln_parameters_bias_ = None + linear = torch._C._nn.linear( + hidden_states, + l_self_modules_linear_parameters_weight_, + l_self_modules_linear_parameters_bias_, + ) + hidden_states = ( + l_self_modules_linear_parameters_weight_ + ) = l_self_modules_linear_parameters_bias_ = None + logits = linear.to(torch.float32) + linear = None + return (logits,) diff --git a/samples/transformers-auto-model/phi-2/weight_meta.py b/samples/transformers-auto-model/phi-2/weight_meta.py new file mode 100644 index 0000000000..6949ddacd4 --- /dev/null +++ b/samples/transformers-auto-model/phi-2/weight_meta.py @@ -0,0 +1,48 @@ +class Program_weight_tensor_meta_L_self_modules_ln_parameters_weight_: + name = "L_self_modules_ln_parameters_weight_" + shape = [2560] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_ln_parameters_bias_: + name = "L_self_modules_ln_parameters_bias_" + shape = [2560] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_hidden_states_: + name = "L_hidden_states_" + shape = [1, 19, 2560] + dtype = "torch.float16" + device = "cpu" + mean = -0.013 + std = 8.773 + data = None + + +class Program_weight_tensor_meta_L_self_modules_linear_parameters_weight_: + name = "L_self_modules_linear_parameters_weight_" + shape = [51200, 2560] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_linear_parameters_bias_: + name = "L_self_modules_linear_parameters_bias_" + shape = [51200] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None diff --git a/samples/transformers-auto-model/tiny_starcoder_py/graph_hash.txt b/samples/transformers-auto-model/tiny_starcoder_py/graph_hash.txt new file mode 100644 index 0000000000..bb0b3a9bcc --- /dev/null +++ b/samples/transformers-auto-model/tiny_starcoder_py/graph_hash.txt @@ -0,0 +1 @@ +02802c2af4db67f72899c45058c5343ebddcd3567ed53593cc3cb8c20fb66af9 \ No newline at end of file diff --git a/samples/transformers-auto-model/tiny_starcoder_py/graph_net.json b/samples/transformers-auto-model/tiny_starcoder_py/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/tiny_starcoder_py/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/tiny_starcoder_py/input_meta.py b/samples/transformers-auto-model/tiny_starcoder_py/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/tiny_starcoder_py/input_tensor_constraints.py b/samples/transformers-auto-model/tiny_starcoder_py/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/tiny_starcoder_py/model.py b/samples/transformers-auto-model/tiny_starcoder_py/model.py new file mode 100644 index 0000000000..f9ccfe2557 --- /dev/null +++ b/samples/transformers-auto-model/tiny_starcoder_py/model.py @@ -0,0 +1,3194 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_self_modules_transformer_modules_wte_parameters_weight_: torch.nn.parameter.Parameter, + L_attention_mask_: torch.Tensor, + L_self_modules_transformer_modules_wpe_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_ln_f_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_transformer_modules_ln_f_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_self_modules_transformer_modules_wte_parameters_weight_ = ( + L_self_modules_transformer_modules_wte_parameters_weight_ + ) + l_attention_mask_ = L_attention_mask_ + l_self_modules_transformer_modules_wpe_parameters_weight_ = ( + L_self_modules_transformer_modules_wpe_parameters_weight_ + ) + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_ = ( + L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_ + ) + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_ + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_ = L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_ + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_ = L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_ + l_self_modules_transformer_modules_ln_f_parameters_weight_ = ( + L_self_modules_transformer_modules_ln_f_parameters_weight_ + ) + l_self_modules_transformer_modules_ln_f_parameters_bias_ = ( + L_self_modules_transformer_modules_ln_f_parameters_bias_ + ) + input_ids = l_input_ids_.view(-1, 19) + l_input_ids_ = None + inputs_embeds = torch.nn.functional.embedding( + input_ids, + l_self_modules_transformer_modules_wte_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + input_ids = None + cache_position = torch.arange(0, 19, device=device(type="cpu")) + position_ids = cache_position.unsqueeze(0) + attention_mask = l_attention_mask_.to( + device=device(type="cpu"), dtype=torch.bool + ) + l_attention_mask_ = None + kv_arange = torch.arange(19, device=device(type="cpu")) + kv_arange += 0 + kv_arange_1 = kv_arange + kv_arange = None + batch_arange = torch.arange(1, device=device(type="cpu")) + head_arange = torch.arange(1, device=device(type="cpu")) + lazy_load_decompositions = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions = None + _vmap_increment_nesting = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting = None + child = torch._C._functorch._add_batch_dim(batch_arange, 0, 1) + batch_arange = None + lazy_load_decompositions_1 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_1 = None + _vmap_increment_nesting_1 = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting_1 = None + child_1 = torch._C._functorch._add_batch_dim(head_arange, 0, 2) + head_arange = child_1 = None + lazy_load_decompositions_2 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_2 = None + _vmap_increment_nesting_2 = torch._C._functorch._vmap_increment_nesting( + 19, "error" + ) + _vmap_increment_nesting_2 = None + child_2 = torch._C._functorch._add_batch_dim(cache_position, 0, 3) + cache_position = None + lazy_load_decompositions_3 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_3 = None + _vmap_increment_nesting_3 = torch._C._functorch._vmap_increment_nesting( + 19, "error" + ) + _vmap_increment_nesting_3 = None + child_3 = torch._C._functorch._add_batch_dim(kv_arange_1, 0, 4) + kv_arange_1 = None + result = child_2.new_ones((), dtype=torch.bool) + le = child_3.le(child_2) + child_2 = None + result_1 = result.__and__(le) + result = le = None + function_ctx = torch.autograd.function.FunctionCtx() + function_ctx = None + index = torch.ops.aten.index(attention_mask, [child, child_3]) + attention_mask = child = child_3 = None + result_2 = result_1.__and__(index) + result_1 = index = None + batched_outputs = torch._C._functorch._remove_batch_dim(result_2, 4, 19, 0) + result_2 = None + _vmap_decrement_nesting = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting = None + batched_outputs_1 = torch._C._functorch._remove_batch_dim( + batched_outputs, 3, 19, 0 + ) + batched_outputs = None + _vmap_decrement_nesting_1 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_1 = None + batched_outputs_2 = torch._C._functorch._remove_batch_dim( + batched_outputs_1, 2, 1, 0 + ) + batched_outputs_1 = None + _vmap_decrement_nesting_2 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_2 = None + causal_mask = torch._C._functorch._remove_batch_dim(batched_outputs_2, 1, 1, 0) + batched_outputs_2 = None + _vmap_decrement_nesting_3 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_3 = None + position_embeds = torch.nn.functional.embedding( + position_ids, + l_self_modules_transformer_modules_wpe_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = l_self_modules_transformer_modules_wpe_parameters_weight_ = None + to_1 = position_embeds.to(device(type="cpu")) + position_embeds = None + hidden_states = inputs_embeds + to_1 + inputs_embeds = to_1 = None + hidden_states_1 = torch.nn.functional.dropout(hidden_states, 0.1, False, False) + hidden_states = None + hidden_states_2 = torch.nn.functional.layer_norm( + hidden_states_1, + (768,), + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_ + ) = None + linear = torch._C._nn.linear( + hidden_states_2, + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_2 = l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_1 = linear.unsqueeze(1) + linear = None + split = unsqueeze_1.split((768, 64, 64), dim=3) + unsqueeze_1 = None + query = split[0] + key = split[1] + value = split[2] + split = None + view_1 = query.view(1, 19, -1, 64) + query = None + query_1 = view_1.transpose(1, 2) + view_1 = None + getitem_3 = key[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key = None + hidden_states_3 = getitem_3.expand(1, 1, 12, 19, 64) + getitem_3 = None + key_1 = hidden_states_3.reshape(1, 12, 19, 64) + hidden_states_3 = None + getitem_4 = value[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value = None + hidden_states_4 = getitem_4.expand(1, 1, 12, 19, 64) + getitem_4 = None + value_1 = hidden_states_4.reshape(1, 12, 19, 64) + hidden_states_4 = None + attention_mask_1 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_2 = query_1.contiguous() + query_1 = None + key_2 = key_1.contiguous() + key_1 = None + value_2 = value_1.contiguous() + value_1 = None + attn_output = torch._C._nn.scaled_dot_product_attention( + query_2, + key_2, + value_2, + attn_mask=attention_mask_1, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_2 = key_2 = value_2 = attention_mask_1 = None + transpose_1 = attn_output.transpose(1, 2) + attn_output = None + attn_output_1 = transpose_1.contiguous() + transpose_1 = None + reshape_2 = attn_output_1.reshape(1, 19, -1) + attn_output_1 = None + attn_output_2 = reshape_2.contiguous() + reshape_2 = None + attn_output_3 = torch._C._nn.linear( + attn_output_2, + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_2 = l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_4 = torch.nn.functional.dropout(attn_output_3, 0.1, False, False) + attn_output_3 = None + hidden_states_5 = attn_output_4 + hidden_states_1 + attn_output_4 = hidden_states_1 = None + hidden_states_6 = torch.nn.functional.layer_norm( + hidden_states_5, + (768,), + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_ + ) = None + hidden_states_7 = torch._C._nn.linear( + hidden_states_6, + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_6 = l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_8 = torch._C._nn.gelu(hidden_states_7, approximate="tanh") + hidden_states_7 = None + hidden_states_9 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_8 = l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_10 = torch.nn.functional.dropout( + hidden_states_9, 0.1, False, False + ) + hidden_states_9 = None + hidden_states_11 = hidden_states_5 + hidden_states_10 + hidden_states_5 = hidden_states_10 = None + hidden_states_12 = torch.nn.functional.layer_norm( + hidden_states_11, + (768,), + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_ + ) = None + linear_4 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_12 = l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_2 = linear_4.unsqueeze(1) + linear_4 = None + split_1 = unsqueeze_2.split((768, 64, 64), dim=3) + unsqueeze_2 = None + query_3 = split_1[0] + key_3 = split_1[1] + value_3 = split_1[2] + split_1 = None + view_2 = query_3.view(1, 19, -1, 64) + query_3 = None + query_4 = view_2.transpose(1, 2) + view_2 = None + getitem_9 = key_3[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_3 = None + hidden_states_13 = getitem_9.expand(1, 1, 12, 19, 64) + getitem_9 = None + key_4 = hidden_states_13.reshape(1, 12, 19, 64) + hidden_states_13 = None + getitem_10 = value_3[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_3 = None + hidden_states_14 = getitem_10.expand(1, 1, 12, 19, 64) + getitem_10 = None + value_4 = hidden_states_14.reshape(1, 12, 19, 64) + hidden_states_14 = None + attention_mask_2 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_5 = query_4.contiguous() + query_4 = None + key_5 = key_4.contiguous() + key_4 = None + value_5 = value_4.contiguous() + value_4 = None + attn_output_5 = torch._C._nn.scaled_dot_product_attention( + query_5, + key_5, + value_5, + attn_mask=attention_mask_2, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_5 = key_5 = value_5 = attention_mask_2 = None + transpose_3 = attn_output_5.transpose(1, 2) + attn_output_5 = None + attn_output_6 = transpose_3.contiguous() + transpose_3 = None + reshape_5 = attn_output_6.reshape(1, 19, -1) + attn_output_6 = None + attn_output_7 = reshape_5.contiguous() + reshape_5 = None + attn_output_8 = torch._C._nn.linear( + attn_output_7, + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_7 = l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_9 = torch.nn.functional.dropout(attn_output_8, 0.1, False, False) + attn_output_8 = None + hidden_states_15 = attn_output_9 + hidden_states_11 + attn_output_9 = hidden_states_11 = None + hidden_states_16 = torch.nn.functional.layer_norm( + hidden_states_15, + (768,), + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_ + ) = None + hidden_states_17 = torch._C._nn.linear( + hidden_states_16, + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_16 = l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_18 = torch._C._nn.gelu(hidden_states_17, approximate="tanh") + hidden_states_17 = None + hidden_states_19 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_18 = l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_20 = torch.nn.functional.dropout( + hidden_states_19, 0.1, False, False + ) + hidden_states_19 = None + hidden_states_21 = hidden_states_15 + hidden_states_20 + hidden_states_15 = hidden_states_20 = None + hidden_states_22 = torch.nn.functional.layer_norm( + hidden_states_21, + (768,), + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_ + ) = None + linear_8 = torch._C._nn.linear( + hidden_states_22, + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_22 = l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_3 = linear_8.unsqueeze(1) + linear_8 = None + split_2 = unsqueeze_3.split((768, 64, 64), dim=3) + unsqueeze_3 = None + query_6 = split_2[0] + key_6 = split_2[1] + value_6 = split_2[2] + split_2 = None + view_3 = query_6.view(1, 19, -1, 64) + query_6 = None + query_7 = view_3.transpose(1, 2) + view_3 = None + getitem_15 = key_6[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_6 = None + hidden_states_23 = getitem_15.expand(1, 1, 12, 19, 64) + getitem_15 = None + key_7 = hidden_states_23.reshape(1, 12, 19, 64) + hidden_states_23 = None + getitem_16 = value_6[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_6 = None + hidden_states_24 = getitem_16.expand(1, 1, 12, 19, 64) + getitem_16 = None + value_7 = hidden_states_24.reshape(1, 12, 19, 64) + hidden_states_24 = None + attention_mask_3 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_8 = query_7.contiguous() + query_7 = None + key_8 = key_7.contiguous() + key_7 = None + value_8 = value_7.contiguous() + value_7 = None + attn_output_10 = torch._C._nn.scaled_dot_product_attention( + query_8, + key_8, + value_8, + attn_mask=attention_mask_3, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_8 = key_8 = value_8 = attention_mask_3 = None + transpose_5 = attn_output_10.transpose(1, 2) + attn_output_10 = None + attn_output_11 = transpose_5.contiguous() + transpose_5 = None + reshape_8 = attn_output_11.reshape(1, 19, -1) + attn_output_11 = None + attn_output_12 = reshape_8.contiguous() + reshape_8 = None + attn_output_13 = torch._C._nn.linear( + attn_output_12, + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_12 = l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_14 = torch.nn.functional.dropout(attn_output_13, 0.1, False, False) + attn_output_13 = None + hidden_states_25 = attn_output_14 + hidden_states_21 + attn_output_14 = hidden_states_21 = None + hidden_states_26 = torch.nn.functional.layer_norm( + hidden_states_25, + (768,), + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_ + ) = None + hidden_states_27 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_26 = l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_28 = torch._C._nn.gelu(hidden_states_27, approximate="tanh") + hidden_states_27 = None + hidden_states_29 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_28 = l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_30 = torch.nn.functional.dropout( + hidden_states_29, 0.1, False, False + ) + hidden_states_29 = None + hidden_states_31 = hidden_states_25 + hidden_states_30 + hidden_states_25 = hidden_states_30 = None + hidden_states_32 = torch.nn.functional.layer_norm( + hidden_states_31, + (768,), + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_ + ) = None + linear_12 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_32 = l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_4 = linear_12.unsqueeze(1) + linear_12 = None + split_3 = unsqueeze_4.split((768, 64, 64), dim=3) + unsqueeze_4 = None + query_9 = split_3[0] + key_9 = split_3[1] + value_9 = split_3[2] + split_3 = None + view_4 = query_9.view(1, 19, -1, 64) + query_9 = None + query_10 = view_4.transpose(1, 2) + view_4 = None + getitem_21 = key_9[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_9 = None + hidden_states_33 = getitem_21.expand(1, 1, 12, 19, 64) + getitem_21 = None + key_10 = hidden_states_33.reshape(1, 12, 19, 64) + hidden_states_33 = None + getitem_22 = value_9[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_9 = None + hidden_states_34 = getitem_22.expand(1, 1, 12, 19, 64) + getitem_22 = None + value_10 = hidden_states_34.reshape(1, 12, 19, 64) + hidden_states_34 = None + attention_mask_4 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_11 = query_10.contiguous() + query_10 = None + key_11 = key_10.contiguous() + key_10 = None + value_11 = value_10.contiguous() + value_10 = None + attn_output_15 = torch._C._nn.scaled_dot_product_attention( + query_11, + key_11, + value_11, + attn_mask=attention_mask_4, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_11 = key_11 = value_11 = attention_mask_4 = None + transpose_7 = attn_output_15.transpose(1, 2) + attn_output_15 = None + attn_output_16 = transpose_7.contiguous() + transpose_7 = None + reshape_11 = attn_output_16.reshape(1, 19, -1) + attn_output_16 = None + attn_output_17 = reshape_11.contiguous() + reshape_11 = None + attn_output_18 = torch._C._nn.linear( + attn_output_17, + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_17 = l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_19 = torch.nn.functional.dropout(attn_output_18, 0.1, False, False) + attn_output_18 = None + hidden_states_35 = attn_output_19 + hidden_states_31 + attn_output_19 = hidden_states_31 = None + hidden_states_36 = torch.nn.functional.layer_norm( + hidden_states_35, + (768,), + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_ + ) = None + hidden_states_37 = torch._C._nn.linear( + hidden_states_36, + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_36 = l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_38 = torch._C._nn.gelu(hidden_states_37, approximate="tanh") + hidden_states_37 = None + hidden_states_39 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_38 = l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_40 = torch.nn.functional.dropout( + hidden_states_39, 0.1, False, False + ) + hidden_states_39 = None + hidden_states_41 = hidden_states_35 + hidden_states_40 + hidden_states_35 = hidden_states_40 = None + hidden_states_42 = torch.nn.functional.layer_norm( + hidden_states_41, + (768,), + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_ + ) = None + linear_16 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_42 = l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_5 = linear_16.unsqueeze(1) + linear_16 = None + split_4 = unsqueeze_5.split((768, 64, 64), dim=3) + unsqueeze_5 = None + query_12 = split_4[0] + key_12 = split_4[1] + value_12 = split_4[2] + split_4 = None + view_5 = query_12.view(1, 19, -1, 64) + query_12 = None + query_13 = view_5.transpose(1, 2) + view_5 = None + getitem_27 = key_12[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_12 = None + hidden_states_43 = getitem_27.expand(1, 1, 12, 19, 64) + getitem_27 = None + key_13 = hidden_states_43.reshape(1, 12, 19, 64) + hidden_states_43 = None + getitem_28 = value_12[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_12 = None + hidden_states_44 = getitem_28.expand(1, 1, 12, 19, 64) + getitem_28 = None + value_13 = hidden_states_44.reshape(1, 12, 19, 64) + hidden_states_44 = None + attention_mask_5 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_14 = query_13.contiguous() + query_13 = None + key_14 = key_13.contiguous() + key_13 = None + value_14 = value_13.contiguous() + value_13 = None + attn_output_20 = torch._C._nn.scaled_dot_product_attention( + query_14, + key_14, + value_14, + attn_mask=attention_mask_5, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_14 = key_14 = value_14 = attention_mask_5 = None + transpose_9 = attn_output_20.transpose(1, 2) + attn_output_20 = None + attn_output_21 = transpose_9.contiguous() + transpose_9 = None + reshape_14 = attn_output_21.reshape(1, 19, -1) + attn_output_21 = None + attn_output_22 = reshape_14.contiguous() + reshape_14 = None + attn_output_23 = torch._C._nn.linear( + attn_output_22, + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_22 = l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_24 = torch.nn.functional.dropout(attn_output_23, 0.1, False, False) + attn_output_23 = None + hidden_states_45 = attn_output_24 + hidden_states_41 + attn_output_24 = hidden_states_41 = None + hidden_states_46 = torch.nn.functional.layer_norm( + hidden_states_45, + (768,), + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_ + ) = None + hidden_states_47 = torch._C._nn.linear( + hidden_states_46, + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_46 = l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_48 = torch._C._nn.gelu(hidden_states_47, approximate="tanh") + hidden_states_47 = None + hidden_states_49 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_48 = l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_50 = torch.nn.functional.dropout( + hidden_states_49, 0.1, False, False + ) + hidden_states_49 = None + hidden_states_51 = hidden_states_45 + hidden_states_50 + hidden_states_45 = hidden_states_50 = None + hidden_states_52 = torch.nn.functional.layer_norm( + hidden_states_51, + (768,), + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_ + ) = None + linear_20 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_52 = l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_6 = linear_20.unsqueeze(1) + linear_20 = None + split_5 = unsqueeze_6.split((768, 64, 64), dim=3) + unsqueeze_6 = None + query_15 = split_5[0] + key_15 = split_5[1] + value_15 = split_5[2] + split_5 = None + view_6 = query_15.view(1, 19, -1, 64) + query_15 = None + query_16 = view_6.transpose(1, 2) + view_6 = None + getitem_33 = key_15[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_15 = None + hidden_states_53 = getitem_33.expand(1, 1, 12, 19, 64) + getitem_33 = None + key_16 = hidden_states_53.reshape(1, 12, 19, 64) + hidden_states_53 = None + getitem_34 = value_15[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_15 = None + hidden_states_54 = getitem_34.expand(1, 1, 12, 19, 64) + getitem_34 = None + value_16 = hidden_states_54.reshape(1, 12, 19, 64) + hidden_states_54 = None + attention_mask_6 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_17 = query_16.contiguous() + query_16 = None + key_17 = key_16.contiguous() + key_16 = None + value_17 = value_16.contiguous() + value_16 = None + attn_output_25 = torch._C._nn.scaled_dot_product_attention( + query_17, + key_17, + value_17, + attn_mask=attention_mask_6, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_17 = key_17 = value_17 = attention_mask_6 = None + transpose_11 = attn_output_25.transpose(1, 2) + attn_output_25 = None + attn_output_26 = transpose_11.contiguous() + transpose_11 = None + reshape_17 = attn_output_26.reshape(1, 19, -1) + attn_output_26 = None + attn_output_27 = reshape_17.contiguous() + reshape_17 = None + attn_output_28 = torch._C._nn.linear( + attn_output_27, + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_27 = l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_29 = torch.nn.functional.dropout(attn_output_28, 0.1, False, False) + attn_output_28 = None + hidden_states_55 = attn_output_29 + hidden_states_51 + attn_output_29 = hidden_states_51 = None + hidden_states_56 = torch.nn.functional.layer_norm( + hidden_states_55, + (768,), + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_ + ) = None + hidden_states_57 = torch._C._nn.linear( + hidden_states_56, + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_56 = l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_58 = torch._C._nn.gelu(hidden_states_57, approximate="tanh") + hidden_states_57 = None + hidden_states_59 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_58 = l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_60 = torch.nn.functional.dropout( + hidden_states_59, 0.1, False, False + ) + hidden_states_59 = None + hidden_states_61 = hidden_states_55 + hidden_states_60 + hidden_states_55 = hidden_states_60 = None + hidden_states_62 = torch.nn.functional.layer_norm( + hidden_states_61, + (768,), + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_ + ) = None + linear_24 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_62 = l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_7 = linear_24.unsqueeze(1) + linear_24 = None + split_6 = unsqueeze_7.split((768, 64, 64), dim=3) + unsqueeze_7 = None + query_18 = split_6[0] + key_18 = split_6[1] + value_18 = split_6[2] + split_6 = None + view_7 = query_18.view(1, 19, -1, 64) + query_18 = None + query_19 = view_7.transpose(1, 2) + view_7 = None + getitem_39 = key_18[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_18 = None + hidden_states_63 = getitem_39.expand(1, 1, 12, 19, 64) + getitem_39 = None + key_19 = hidden_states_63.reshape(1, 12, 19, 64) + hidden_states_63 = None + getitem_40 = value_18[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_18 = None + hidden_states_64 = getitem_40.expand(1, 1, 12, 19, 64) + getitem_40 = None + value_19 = hidden_states_64.reshape(1, 12, 19, 64) + hidden_states_64 = None + attention_mask_7 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_20 = query_19.contiguous() + query_19 = None + key_20 = key_19.contiguous() + key_19 = None + value_20 = value_19.contiguous() + value_19 = None + attn_output_30 = torch._C._nn.scaled_dot_product_attention( + query_20, + key_20, + value_20, + attn_mask=attention_mask_7, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_20 = key_20 = value_20 = attention_mask_7 = None + transpose_13 = attn_output_30.transpose(1, 2) + attn_output_30 = None + attn_output_31 = transpose_13.contiguous() + transpose_13 = None + reshape_20 = attn_output_31.reshape(1, 19, -1) + attn_output_31 = None + attn_output_32 = reshape_20.contiguous() + reshape_20 = None + attn_output_33 = torch._C._nn.linear( + attn_output_32, + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_32 = l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_34 = torch.nn.functional.dropout(attn_output_33, 0.1, False, False) + attn_output_33 = None + hidden_states_65 = attn_output_34 + hidden_states_61 + attn_output_34 = hidden_states_61 = None + hidden_states_66 = torch.nn.functional.layer_norm( + hidden_states_65, + (768,), + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_ + ) = None + hidden_states_67 = torch._C._nn.linear( + hidden_states_66, + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_66 = l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_68 = torch._C._nn.gelu(hidden_states_67, approximate="tanh") + hidden_states_67 = None + hidden_states_69 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_68 = l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_70 = torch.nn.functional.dropout( + hidden_states_69, 0.1, False, False + ) + hidden_states_69 = None + hidden_states_71 = hidden_states_65 + hidden_states_70 + hidden_states_65 = hidden_states_70 = None + hidden_states_72 = torch.nn.functional.layer_norm( + hidden_states_71, + (768,), + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_ + ) = None + linear_28 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_72 = l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_8 = linear_28.unsqueeze(1) + linear_28 = None + split_7 = unsqueeze_8.split((768, 64, 64), dim=3) + unsqueeze_8 = None + query_21 = split_7[0] + key_21 = split_7[1] + value_21 = split_7[2] + split_7 = None + view_8 = query_21.view(1, 19, -1, 64) + query_21 = None + query_22 = view_8.transpose(1, 2) + view_8 = None + getitem_45 = key_21[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_21 = None + hidden_states_73 = getitem_45.expand(1, 1, 12, 19, 64) + getitem_45 = None + key_22 = hidden_states_73.reshape(1, 12, 19, 64) + hidden_states_73 = None + getitem_46 = value_21[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_21 = None + hidden_states_74 = getitem_46.expand(1, 1, 12, 19, 64) + getitem_46 = None + value_22 = hidden_states_74.reshape(1, 12, 19, 64) + hidden_states_74 = None + attention_mask_8 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_23 = query_22.contiguous() + query_22 = None + key_23 = key_22.contiguous() + key_22 = None + value_23 = value_22.contiguous() + value_22 = None + attn_output_35 = torch._C._nn.scaled_dot_product_attention( + query_23, + key_23, + value_23, + attn_mask=attention_mask_8, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_23 = key_23 = value_23 = attention_mask_8 = None + transpose_15 = attn_output_35.transpose(1, 2) + attn_output_35 = None + attn_output_36 = transpose_15.contiguous() + transpose_15 = None + reshape_23 = attn_output_36.reshape(1, 19, -1) + attn_output_36 = None + attn_output_37 = reshape_23.contiguous() + reshape_23 = None + attn_output_38 = torch._C._nn.linear( + attn_output_37, + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_37 = l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_39 = torch.nn.functional.dropout(attn_output_38, 0.1, False, False) + attn_output_38 = None + hidden_states_75 = attn_output_39 + hidden_states_71 + attn_output_39 = hidden_states_71 = None + hidden_states_76 = torch.nn.functional.layer_norm( + hidden_states_75, + (768,), + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_ + ) = None + hidden_states_77 = torch._C._nn.linear( + hidden_states_76, + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_76 = l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_78 = torch._C._nn.gelu(hidden_states_77, approximate="tanh") + hidden_states_77 = None + hidden_states_79 = torch._C._nn.linear( + hidden_states_78, + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_78 = l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_80 = torch.nn.functional.dropout( + hidden_states_79, 0.1, False, False + ) + hidden_states_79 = None + hidden_states_81 = hidden_states_75 + hidden_states_80 + hidden_states_75 = hidden_states_80 = None + hidden_states_82 = torch.nn.functional.layer_norm( + hidden_states_81, + (768,), + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_ + ) = None + linear_32 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_82 = l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_9 = linear_32.unsqueeze(1) + linear_32 = None + split_8 = unsqueeze_9.split((768, 64, 64), dim=3) + unsqueeze_9 = None + query_24 = split_8[0] + key_24 = split_8[1] + value_24 = split_8[2] + split_8 = None + view_9 = query_24.view(1, 19, -1, 64) + query_24 = None + query_25 = view_9.transpose(1, 2) + view_9 = None + getitem_51 = key_24[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_24 = None + hidden_states_83 = getitem_51.expand(1, 1, 12, 19, 64) + getitem_51 = None + key_25 = hidden_states_83.reshape(1, 12, 19, 64) + hidden_states_83 = None + getitem_52 = value_24[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_24 = None + hidden_states_84 = getitem_52.expand(1, 1, 12, 19, 64) + getitem_52 = None + value_25 = hidden_states_84.reshape(1, 12, 19, 64) + hidden_states_84 = None + attention_mask_9 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_26 = query_25.contiguous() + query_25 = None + key_26 = key_25.contiguous() + key_25 = None + value_26 = value_25.contiguous() + value_25 = None + attn_output_40 = torch._C._nn.scaled_dot_product_attention( + query_26, + key_26, + value_26, + attn_mask=attention_mask_9, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_26 = key_26 = value_26 = attention_mask_9 = None + transpose_17 = attn_output_40.transpose(1, 2) + attn_output_40 = None + attn_output_41 = transpose_17.contiguous() + transpose_17 = None + reshape_26 = attn_output_41.reshape(1, 19, -1) + attn_output_41 = None + attn_output_42 = reshape_26.contiguous() + reshape_26 = None + attn_output_43 = torch._C._nn.linear( + attn_output_42, + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_42 = l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_44 = torch.nn.functional.dropout(attn_output_43, 0.1, False, False) + attn_output_43 = None + hidden_states_85 = attn_output_44 + hidden_states_81 + attn_output_44 = hidden_states_81 = None + hidden_states_86 = torch.nn.functional.layer_norm( + hidden_states_85, + (768,), + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_ + ) = None + hidden_states_87 = torch._C._nn.linear( + hidden_states_86, + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_86 = l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_88 = torch._C._nn.gelu(hidden_states_87, approximate="tanh") + hidden_states_87 = None + hidden_states_89 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_88 = l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_90 = torch.nn.functional.dropout( + hidden_states_89, 0.1, False, False + ) + hidden_states_89 = None + hidden_states_91 = hidden_states_85 + hidden_states_90 + hidden_states_85 = hidden_states_90 = None + hidden_states_92 = torch.nn.functional.layer_norm( + hidden_states_91, + (768,), + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_ + ) = None + linear_36 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_92 = l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_10 = linear_36.unsqueeze(1) + linear_36 = None + split_9 = unsqueeze_10.split((768, 64, 64), dim=3) + unsqueeze_10 = None + query_27 = split_9[0] + key_27 = split_9[1] + value_27 = split_9[2] + split_9 = None + view_10 = query_27.view(1, 19, -1, 64) + query_27 = None + query_28 = view_10.transpose(1, 2) + view_10 = None + getitem_57 = key_27[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_27 = None + hidden_states_93 = getitem_57.expand(1, 1, 12, 19, 64) + getitem_57 = None + key_28 = hidden_states_93.reshape(1, 12, 19, 64) + hidden_states_93 = None + getitem_58 = value_27[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_27 = None + hidden_states_94 = getitem_58.expand(1, 1, 12, 19, 64) + getitem_58 = None + value_28 = hidden_states_94.reshape(1, 12, 19, 64) + hidden_states_94 = None + attention_mask_10 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_29 = query_28.contiguous() + query_28 = None + key_29 = key_28.contiguous() + key_28 = None + value_29 = value_28.contiguous() + value_28 = None + attn_output_45 = torch._C._nn.scaled_dot_product_attention( + query_29, + key_29, + value_29, + attn_mask=attention_mask_10, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_29 = key_29 = value_29 = attention_mask_10 = None + transpose_19 = attn_output_45.transpose(1, 2) + attn_output_45 = None + attn_output_46 = transpose_19.contiguous() + transpose_19 = None + reshape_29 = attn_output_46.reshape(1, 19, -1) + attn_output_46 = None + attn_output_47 = reshape_29.contiguous() + reshape_29 = None + attn_output_48 = torch._C._nn.linear( + attn_output_47, + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_47 = l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_49 = torch.nn.functional.dropout(attn_output_48, 0.1, False, False) + attn_output_48 = None + hidden_states_95 = attn_output_49 + hidden_states_91 + attn_output_49 = hidden_states_91 = None + hidden_states_96 = torch.nn.functional.layer_norm( + hidden_states_95, + (768,), + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_ = ( + l_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_ + ) = None + hidden_states_97 = torch._C._nn.linear( + hidden_states_96, + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_96 = l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_98 = torch._C._nn.gelu(hidden_states_97, approximate="tanh") + hidden_states_97 = None + hidden_states_99 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_98 = l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_100 = torch.nn.functional.dropout( + hidden_states_99, 0.1, False, False + ) + hidden_states_99 = None + hidden_states_101 = hidden_states_95 + hidden_states_100 + hidden_states_95 = hidden_states_100 = None + hidden_states_102 = torch.nn.functional.layer_norm( + hidden_states_101, + (768,), + l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_ = (None) + linear_40 = torch._C._nn.linear( + hidden_states_102, + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_102 = l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_11 = linear_40.unsqueeze(1) + linear_40 = None + split_10 = unsqueeze_11.split((768, 64, 64), dim=3) + unsqueeze_11 = None + query_30 = split_10[0] + key_30 = split_10[1] + value_30 = split_10[2] + split_10 = None + view_11 = query_30.view(1, 19, -1, 64) + query_30 = None + query_31 = view_11.transpose(1, 2) + view_11 = None + getitem_63 = key_30[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_30 = None + hidden_states_103 = getitem_63.expand(1, 1, 12, 19, 64) + getitem_63 = None + key_31 = hidden_states_103.reshape(1, 12, 19, 64) + hidden_states_103 = None + getitem_64 = value_30[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_30 = None + hidden_states_104 = getitem_64.expand(1, 1, 12, 19, 64) + getitem_64 = None + value_31 = hidden_states_104.reshape(1, 12, 19, 64) + hidden_states_104 = None + attention_mask_11 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_32 = query_31.contiguous() + query_31 = None + key_32 = key_31.contiguous() + key_31 = None + value_32 = value_31.contiguous() + value_31 = None + attn_output_50 = torch._C._nn.scaled_dot_product_attention( + query_32, + key_32, + value_32, + attn_mask=attention_mask_11, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_32 = key_32 = value_32 = attention_mask_11 = None + transpose_21 = attn_output_50.transpose(1, 2) + attn_output_50 = None + attn_output_51 = transpose_21.contiguous() + transpose_21 = None + reshape_32 = attn_output_51.reshape(1, 19, -1) + attn_output_51 = None + attn_output_52 = reshape_32.contiguous() + reshape_32 = None + attn_output_53 = torch._C._nn.linear( + attn_output_52, + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_52 = l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_54 = torch.nn.functional.dropout(attn_output_53, 0.1, False, False) + attn_output_53 = None + hidden_states_105 = attn_output_54 + hidden_states_101 + attn_output_54 = hidden_states_101 = None + hidden_states_106 = torch.nn.functional.layer_norm( + hidden_states_105, + (768,), + l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_ = (None) + hidden_states_107 = torch._C._nn.linear( + hidden_states_106, + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_106 = l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_108 = torch._C._nn.gelu(hidden_states_107, approximate="tanh") + hidden_states_107 = None + hidden_states_109 = torch._C._nn.linear( + hidden_states_108, + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_108 = l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_110 = torch.nn.functional.dropout( + hidden_states_109, 0.1, False, False + ) + hidden_states_109 = None + hidden_states_111 = hidden_states_105 + hidden_states_110 + hidden_states_105 = hidden_states_110 = None + hidden_states_112 = torch.nn.functional.layer_norm( + hidden_states_111, + (768,), + l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_ = (None) + linear_44 = torch._C._nn.linear( + hidden_states_112, + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_112 = l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_12 = linear_44.unsqueeze(1) + linear_44 = None + split_11 = unsqueeze_12.split((768, 64, 64), dim=3) + unsqueeze_12 = None + query_33 = split_11[0] + key_33 = split_11[1] + value_33 = split_11[2] + split_11 = None + view_12 = query_33.view(1, 19, -1, 64) + query_33 = None + query_34 = view_12.transpose(1, 2) + view_12 = None + getitem_69 = key_33[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_33 = None + hidden_states_113 = getitem_69.expand(1, 1, 12, 19, 64) + getitem_69 = None + key_34 = hidden_states_113.reshape(1, 12, 19, 64) + hidden_states_113 = None + getitem_70 = value_33[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_33 = None + hidden_states_114 = getitem_70.expand(1, 1, 12, 19, 64) + getitem_70 = None + value_34 = hidden_states_114.reshape(1, 12, 19, 64) + hidden_states_114 = None + attention_mask_12 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_35 = query_34.contiguous() + query_34 = None + key_35 = key_34.contiguous() + key_34 = None + value_35 = value_34.contiguous() + value_34 = None + attn_output_55 = torch._C._nn.scaled_dot_product_attention( + query_35, + key_35, + value_35, + attn_mask=attention_mask_12, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_35 = key_35 = value_35 = attention_mask_12 = None + transpose_23 = attn_output_55.transpose(1, 2) + attn_output_55 = None + attn_output_56 = transpose_23.contiguous() + transpose_23 = None + reshape_35 = attn_output_56.reshape(1, 19, -1) + attn_output_56 = None + attn_output_57 = reshape_35.contiguous() + reshape_35 = None + attn_output_58 = torch._C._nn.linear( + attn_output_57, + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_57 = l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_59 = torch.nn.functional.dropout(attn_output_58, 0.1, False, False) + attn_output_58 = None + hidden_states_115 = attn_output_59 + hidden_states_111 + attn_output_59 = hidden_states_111 = None + hidden_states_116 = torch.nn.functional.layer_norm( + hidden_states_115, + (768,), + l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_ = (None) + hidden_states_117 = torch._C._nn.linear( + hidden_states_116, + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_116 = l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_118 = torch._C._nn.gelu(hidden_states_117, approximate="tanh") + hidden_states_117 = None + hidden_states_119 = torch._C._nn.linear( + hidden_states_118, + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_118 = l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_120 = torch.nn.functional.dropout( + hidden_states_119, 0.1, False, False + ) + hidden_states_119 = None + hidden_states_121 = hidden_states_115 + hidden_states_120 + hidden_states_115 = hidden_states_120 = None + hidden_states_122 = torch.nn.functional.layer_norm( + hidden_states_121, + (768,), + l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_ = (None) + linear_48 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_122 = l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_13 = linear_48.unsqueeze(1) + linear_48 = None + split_12 = unsqueeze_13.split((768, 64, 64), dim=3) + unsqueeze_13 = None + query_36 = split_12[0] + key_36 = split_12[1] + value_36 = split_12[2] + split_12 = None + view_13 = query_36.view(1, 19, -1, 64) + query_36 = None + query_37 = view_13.transpose(1, 2) + view_13 = None + getitem_75 = key_36[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_36 = None + hidden_states_123 = getitem_75.expand(1, 1, 12, 19, 64) + getitem_75 = None + key_37 = hidden_states_123.reshape(1, 12, 19, 64) + hidden_states_123 = None + getitem_76 = value_36[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_36 = None + hidden_states_124 = getitem_76.expand(1, 1, 12, 19, 64) + getitem_76 = None + value_37 = hidden_states_124.reshape(1, 12, 19, 64) + hidden_states_124 = None + attention_mask_13 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_38 = query_37.contiguous() + query_37 = None + key_38 = key_37.contiguous() + key_37 = None + value_38 = value_37.contiguous() + value_37 = None + attn_output_60 = torch._C._nn.scaled_dot_product_attention( + query_38, + key_38, + value_38, + attn_mask=attention_mask_13, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_38 = key_38 = value_38 = attention_mask_13 = None + transpose_25 = attn_output_60.transpose(1, 2) + attn_output_60 = None + attn_output_61 = transpose_25.contiguous() + transpose_25 = None + reshape_38 = attn_output_61.reshape(1, 19, -1) + attn_output_61 = None + attn_output_62 = reshape_38.contiguous() + reshape_38 = None + attn_output_63 = torch._C._nn.linear( + attn_output_62, + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_62 = l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_64 = torch.nn.functional.dropout(attn_output_63, 0.1, False, False) + attn_output_63 = None + hidden_states_125 = attn_output_64 + hidden_states_121 + attn_output_64 = hidden_states_121 = None + hidden_states_126 = torch.nn.functional.layer_norm( + hidden_states_125, + (768,), + l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_ = (None) + hidden_states_127 = torch._C._nn.linear( + hidden_states_126, + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_126 = l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_128 = torch._C._nn.gelu(hidden_states_127, approximate="tanh") + hidden_states_127 = None + hidden_states_129 = torch._C._nn.linear( + hidden_states_128, + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_128 = l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_130 = torch.nn.functional.dropout( + hidden_states_129, 0.1, False, False + ) + hidden_states_129 = None + hidden_states_131 = hidden_states_125 + hidden_states_130 + hidden_states_125 = hidden_states_130 = None + hidden_states_132 = torch.nn.functional.layer_norm( + hidden_states_131, + (768,), + l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_ = (None) + linear_52 = torch._C._nn.linear( + hidden_states_132, + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_132 = l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_14 = linear_52.unsqueeze(1) + linear_52 = None + split_13 = unsqueeze_14.split((768, 64, 64), dim=3) + unsqueeze_14 = None + query_39 = split_13[0] + key_39 = split_13[1] + value_39 = split_13[2] + split_13 = None + view_14 = query_39.view(1, 19, -1, 64) + query_39 = None + query_40 = view_14.transpose(1, 2) + view_14 = None + getitem_81 = key_39[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_39 = None + hidden_states_133 = getitem_81.expand(1, 1, 12, 19, 64) + getitem_81 = None + key_40 = hidden_states_133.reshape(1, 12, 19, 64) + hidden_states_133 = None + getitem_82 = value_39[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_39 = None + hidden_states_134 = getitem_82.expand(1, 1, 12, 19, 64) + getitem_82 = None + value_40 = hidden_states_134.reshape(1, 12, 19, 64) + hidden_states_134 = None + attention_mask_14 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_41 = query_40.contiguous() + query_40 = None + key_41 = key_40.contiguous() + key_40 = None + value_41 = value_40.contiguous() + value_40 = None + attn_output_65 = torch._C._nn.scaled_dot_product_attention( + query_41, + key_41, + value_41, + attn_mask=attention_mask_14, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_41 = key_41 = value_41 = attention_mask_14 = None + transpose_27 = attn_output_65.transpose(1, 2) + attn_output_65 = None + attn_output_66 = transpose_27.contiguous() + transpose_27 = None + reshape_41 = attn_output_66.reshape(1, 19, -1) + attn_output_66 = None + attn_output_67 = reshape_41.contiguous() + reshape_41 = None + attn_output_68 = torch._C._nn.linear( + attn_output_67, + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_67 = l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_69 = torch.nn.functional.dropout(attn_output_68, 0.1, False, False) + attn_output_68 = None + hidden_states_135 = attn_output_69 + hidden_states_131 + attn_output_69 = hidden_states_131 = None + hidden_states_136 = torch.nn.functional.layer_norm( + hidden_states_135, + (768,), + l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_ = (None) + hidden_states_137 = torch._C._nn.linear( + hidden_states_136, + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_136 = l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_138 = torch._C._nn.gelu(hidden_states_137, approximate="tanh") + hidden_states_137 = None + hidden_states_139 = torch._C._nn.linear( + hidden_states_138, + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_138 = l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_140 = torch.nn.functional.dropout( + hidden_states_139, 0.1, False, False + ) + hidden_states_139 = None + hidden_states_141 = hidden_states_135 + hidden_states_140 + hidden_states_135 = hidden_states_140 = None + hidden_states_142 = torch.nn.functional.layer_norm( + hidden_states_141, + (768,), + l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_ = (None) + linear_56 = torch._C._nn.linear( + hidden_states_142, + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_142 = l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_15 = linear_56.unsqueeze(1) + linear_56 = None + split_14 = unsqueeze_15.split((768, 64, 64), dim=3) + unsqueeze_15 = None + query_42 = split_14[0] + key_42 = split_14[1] + value_42 = split_14[2] + split_14 = None + view_15 = query_42.view(1, 19, -1, 64) + query_42 = None + query_43 = view_15.transpose(1, 2) + view_15 = None + getitem_87 = key_42[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_42 = None + hidden_states_143 = getitem_87.expand(1, 1, 12, 19, 64) + getitem_87 = None + key_43 = hidden_states_143.reshape(1, 12, 19, 64) + hidden_states_143 = None + getitem_88 = value_42[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_42 = None + hidden_states_144 = getitem_88.expand(1, 1, 12, 19, 64) + getitem_88 = None + value_43 = hidden_states_144.reshape(1, 12, 19, 64) + hidden_states_144 = None + attention_mask_15 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_44 = query_43.contiguous() + query_43 = None + key_44 = key_43.contiguous() + key_43 = None + value_44 = value_43.contiguous() + value_43 = None + attn_output_70 = torch._C._nn.scaled_dot_product_attention( + query_44, + key_44, + value_44, + attn_mask=attention_mask_15, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_44 = key_44 = value_44 = attention_mask_15 = None + transpose_29 = attn_output_70.transpose(1, 2) + attn_output_70 = None + attn_output_71 = transpose_29.contiguous() + transpose_29 = None + reshape_44 = attn_output_71.reshape(1, 19, -1) + attn_output_71 = None + attn_output_72 = reshape_44.contiguous() + reshape_44 = None + attn_output_73 = torch._C._nn.linear( + attn_output_72, + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_72 = l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_74 = torch.nn.functional.dropout(attn_output_73, 0.1, False, False) + attn_output_73 = None + hidden_states_145 = attn_output_74 + hidden_states_141 + attn_output_74 = hidden_states_141 = None + hidden_states_146 = torch.nn.functional.layer_norm( + hidden_states_145, + (768,), + l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_ = (None) + hidden_states_147 = torch._C._nn.linear( + hidden_states_146, + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_146 = l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_148 = torch._C._nn.gelu(hidden_states_147, approximate="tanh") + hidden_states_147 = None + hidden_states_149 = torch._C._nn.linear( + hidden_states_148, + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_148 = l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_150 = torch.nn.functional.dropout( + hidden_states_149, 0.1, False, False + ) + hidden_states_149 = None + hidden_states_151 = hidden_states_145 + hidden_states_150 + hidden_states_145 = hidden_states_150 = None + hidden_states_152 = torch.nn.functional.layer_norm( + hidden_states_151, + (768,), + l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_ = (None) + linear_60 = torch._C._nn.linear( + hidden_states_152, + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_152 = l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_16 = linear_60.unsqueeze(1) + linear_60 = None + split_15 = unsqueeze_16.split((768, 64, 64), dim=3) + unsqueeze_16 = None + query_45 = split_15[0] + key_45 = split_15[1] + value_45 = split_15[2] + split_15 = None + view_16 = query_45.view(1, 19, -1, 64) + query_45 = None + query_46 = view_16.transpose(1, 2) + view_16 = None + getitem_93 = key_45[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_45 = None + hidden_states_153 = getitem_93.expand(1, 1, 12, 19, 64) + getitem_93 = None + key_46 = hidden_states_153.reshape(1, 12, 19, 64) + hidden_states_153 = None + getitem_94 = value_45[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_45 = None + hidden_states_154 = getitem_94.expand(1, 1, 12, 19, 64) + getitem_94 = None + value_46 = hidden_states_154.reshape(1, 12, 19, 64) + hidden_states_154 = None + attention_mask_16 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_47 = query_46.contiguous() + query_46 = None + key_47 = key_46.contiguous() + key_46 = None + value_47 = value_46.contiguous() + value_46 = None + attn_output_75 = torch._C._nn.scaled_dot_product_attention( + query_47, + key_47, + value_47, + attn_mask=attention_mask_16, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_47 = key_47 = value_47 = attention_mask_16 = None + transpose_31 = attn_output_75.transpose(1, 2) + attn_output_75 = None + attn_output_76 = transpose_31.contiguous() + transpose_31 = None + reshape_47 = attn_output_76.reshape(1, 19, -1) + attn_output_76 = None + attn_output_77 = reshape_47.contiguous() + reshape_47 = None + attn_output_78 = torch._C._nn.linear( + attn_output_77, + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_77 = l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_79 = torch.nn.functional.dropout(attn_output_78, 0.1, False, False) + attn_output_78 = None + hidden_states_155 = attn_output_79 + hidden_states_151 + attn_output_79 = hidden_states_151 = None + hidden_states_156 = torch.nn.functional.layer_norm( + hidden_states_155, + (768,), + l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_ = (None) + hidden_states_157 = torch._C._nn.linear( + hidden_states_156, + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_156 = l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_158 = torch._C._nn.gelu(hidden_states_157, approximate="tanh") + hidden_states_157 = None + hidden_states_159 = torch._C._nn.linear( + hidden_states_158, + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_158 = l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_160 = torch.nn.functional.dropout( + hidden_states_159, 0.1, False, False + ) + hidden_states_159 = None + hidden_states_161 = hidden_states_155 + hidden_states_160 + hidden_states_155 = hidden_states_160 = None + hidden_states_162 = torch.nn.functional.layer_norm( + hidden_states_161, + (768,), + l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_ = (None) + linear_64 = torch._C._nn.linear( + hidden_states_162, + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_162 = l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_17 = linear_64.unsqueeze(1) + linear_64 = None + split_16 = unsqueeze_17.split((768, 64, 64), dim=3) + unsqueeze_17 = None + query_48 = split_16[0] + key_48 = split_16[1] + value_48 = split_16[2] + split_16 = None + view_17 = query_48.view(1, 19, -1, 64) + query_48 = None + query_49 = view_17.transpose(1, 2) + view_17 = None + getitem_99 = key_48[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_48 = None + hidden_states_163 = getitem_99.expand(1, 1, 12, 19, 64) + getitem_99 = None + key_49 = hidden_states_163.reshape(1, 12, 19, 64) + hidden_states_163 = None + getitem_100 = value_48[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_48 = None + hidden_states_164 = getitem_100.expand(1, 1, 12, 19, 64) + getitem_100 = None + value_49 = hidden_states_164.reshape(1, 12, 19, 64) + hidden_states_164 = None + attention_mask_17 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_50 = query_49.contiguous() + query_49 = None + key_50 = key_49.contiguous() + key_49 = None + value_50 = value_49.contiguous() + value_49 = None + attn_output_80 = torch._C._nn.scaled_dot_product_attention( + query_50, + key_50, + value_50, + attn_mask=attention_mask_17, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_50 = key_50 = value_50 = attention_mask_17 = None + transpose_33 = attn_output_80.transpose(1, 2) + attn_output_80 = None + attn_output_81 = transpose_33.contiguous() + transpose_33 = None + reshape_50 = attn_output_81.reshape(1, 19, -1) + attn_output_81 = None + attn_output_82 = reshape_50.contiguous() + reshape_50 = None + attn_output_83 = torch._C._nn.linear( + attn_output_82, + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_82 = l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_84 = torch.nn.functional.dropout(attn_output_83, 0.1, False, False) + attn_output_83 = None + hidden_states_165 = attn_output_84 + hidden_states_161 + attn_output_84 = hidden_states_161 = None + hidden_states_166 = torch.nn.functional.layer_norm( + hidden_states_165, + (768,), + l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_ = (None) + hidden_states_167 = torch._C._nn.linear( + hidden_states_166, + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_166 = l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_168 = torch._C._nn.gelu(hidden_states_167, approximate="tanh") + hidden_states_167 = None + hidden_states_169 = torch._C._nn.linear( + hidden_states_168, + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_168 = l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_170 = torch.nn.functional.dropout( + hidden_states_169, 0.1, False, False + ) + hidden_states_169 = None + hidden_states_171 = hidden_states_165 + hidden_states_170 + hidden_states_165 = hidden_states_170 = None + hidden_states_172 = torch.nn.functional.layer_norm( + hidden_states_171, + (768,), + l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_ = (None) + linear_68 = torch._C._nn.linear( + hidden_states_172, + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_172 = l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_18 = linear_68.unsqueeze(1) + linear_68 = None + split_17 = unsqueeze_18.split((768, 64, 64), dim=3) + unsqueeze_18 = None + query_51 = split_17[0] + key_51 = split_17[1] + value_51 = split_17[2] + split_17 = None + view_18 = query_51.view(1, 19, -1, 64) + query_51 = None + query_52 = view_18.transpose(1, 2) + view_18 = None + getitem_105 = key_51[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_51 = None + hidden_states_173 = getitem_105.expand(1, 1, 12, 19, 64) + getitem_105 = None + key_52 = hidden_states_173.reshape(1, 12, 19, 64) + hidden_states_173 = None + getitem_106 = value_51[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_51 = None + hidden_states_174 = getitem_106.expand(1, 1, 12, 19, 64) + getitem_106 = None + value_52 = hidden_states_174.reshape(1, 12, 19, 64) + hidden_states_174 = None + attention_mask_18 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_53 = query_52.contiguous() + query_52 = None + key_53 = key_52.contiguous() + key_52 = None + value_53 = value_52.contiguous() + value_52 = None + attn_output_85 = torch._C._nn.scaled_dot_product_attention( + query_53, + key_53, + value_53, + attn_mask=attention_mask_18, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_53 = key_53 = value_53 = attention_mask_18 = None + transpose_35 = attn_output_85.transpose(1, 2) + attn_output_85 = None + attn_output_86 = transpose_35.contiguous() + transpose_35 = None + reshape_53 = attn_output_86.reshape(1, 19, -1) + attn_output_86 = None + attn_output_87 = reshape_53.contiguous() + reshape_53 = None + attn_output_88 = torch._C._nn.linear( + attn_output_87, + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_87 = l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_89 = torch.nn.functional.dropout(attn_output_88, 0.1, False, False) + attn_output_88 = None + hidden_states_175 = attn_output_89 + hidden_states_171 + attn_output_89 = hidden_states_171 = None + hidden_states_176 = torch.nn.functional.layer_norm( + hidden_states_175, + (768,), + l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_ = (None) + hidden_states_177 = torch._C._nn.linear( + hidden_states_176, + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_176 = l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_178 = torch._C._nn.gelu(hidden_states_177, approximate="tanh") + hidden_states_177 = None + hidden_states_179 = torch._C._nn.linear( + hidden_states_178, + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_178 = l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_180 = torch.nn.functional.dropout( + hidden_states_179, 0.1, False, False + ) + hidden_states_179 = None + hidden_states_181 = hidden_states_175 + hidden_states_180 + hidden_states_175 = hidden_states_180 = None + hidden_states_182 = torch.nn.functional.layer_norm( + hidden_states_181, + (768,), + l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_ = (None) + linear_72 = torch._C._nn.linear( + hidden_states_182, + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_182 = l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_19 = linear_72.unsqueeze(1) + linear_72 = None + split_18 = unsqueeze_19.split((768, 64, 64), dim=3) + unsqueeze_19 = None + query_54 = split_18[0] + key_54 = split_18[1] + value_54 = split_18[2] + split_18 = None + view_19 = query_54.view(1, 19, -1, 64) + query_54 = None + query_55 = view_19.transpose(1, 2) + view_19 = None + getitem_111 = key_54[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_54 = None + hidden_states_183 = getitem_111.expand(1, 1, 12, 19, 64) + getitem_111 = None + key_55 = hidden_states_183.reshape(1, 12, 19, 64) + hidden_states_183 = None + getitem_112 = value_54[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_54 = None + hidden_states_184 = getitem_112.expand(1, 1, 12, 19, 64) + getitem_112 = None + value_55 = hidden_states_184.reshape(1, 12, 19, 64) + hidden_states_184 = None + attention_mask_19 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + query_56 = query_55.contiguous() + query_55 = None + key_56 = key_55.contiguous() + key_55 = None + value_56 = value_55.contiguous() + value_55 = None + attn_output_90 = torch._C._nn.scaled_dot_product_attention( + query_56, + key_56, + value_56, + attn_mask=attention_mask_19, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_56 = key_56 = value_56 = attention_mask_19 = None + transpose_37 = attn_output_90.transpose(1, 2) + attn_output_90 = None + attn_output_91 = transpose_37.contiguous() + transpose_37 = None + reshape_56 = attn_output_91.reshape(1, 19, -1) + attn_output_91 = None + attn_output_92 = reshape_56.contiguous() + reshape_56 = None + attn_output_93 = torch._C._nn.linear( + attn_output_92, + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_92 = l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_94 = torch.nn.functional.dropout(attn_output_93, 0.1, False, False) + attn_output_93 = None + hidden_states_185 = attn_output_94 + hidden_states_181 + attn_output_94 = hidden_states_181 = None + hidden_states_186 = torch.nn.functional.layer_norm( + hidden_states_185, + (768,), + l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_ = (None) + hidden_states_187 = torch._C._nn.linear( + hidden_states_186, + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_186 = l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_188 = torch._C._nn.gelu(hidden_states_187, approximate="tanh") + hidden_states_187 = None + hidden_states_189 = torch._C._nn.linear( + hidden_states_188, + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_188 = l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_190 = torch.nn.functional.dropout( + hidden_states_189, 0.1, False, False + ) + hidden_states_189 = None + hidden_states_191 = hidden_states_185 + hidden_states_190 + hidden_states_185 = hidden_states_190 = None + hidden_states_192 = torch.nn.functional.layer_norm( + hidden_states_191, + (768,), + l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_ = (None) + linear_76 = torch._C._nn.linear( + hidden_states_192, + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_, + ) + hidden_states_192 = l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_ = (None) + unsqueeze_20 = linear_76.unsqueeze(1) + linear_76 = None + split_19 = unsqueeze_20.split((768, 64, 64), dim=3) + unsqueeze_20 = None + query_57 = split_19[0] + key_57 = split_19[1] + value_57 = split_19[2] + split_19 = None + view_20 = query_57.view(1, 19, -1, 64) + query_57 = None + query_58 = view_20.transpose(1, 2) + view_20 = None + getitem_117 = key_57[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + key_57 = None + hidden_states_193 = getitem_117.expand(1, 1, 12, 19, 64) + getitem_117 = None + key_58 = hidden_states_193.reshape(1, 12, 19, 64) + hidden_states_193 = None + getitem_118 = value_57[ + ( + slice(None, None, None), + slice(None, None, None), + None, + slice(None, None, None), + slice(None, None, None), + ) + ] + value_57 = None + hidden_states_194 = getitem_118.expand(1, 1, 12, 19, 64) + getitem_118 = None + value_58 = hidden_states_194.reshape(1, 12, 19, 64) + hidden_states_194 = None + attention_mask_20 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 19, None), + ) + ] + causal_mask = None + query_59 = query_58.contiguous() + query_58 = None + key_59 = key_58.contiguous() + key_58 = None + value_59 = value_58.contiguous() + value_58 = None + attn_output_95 = torch._C._nn.scaled_dot_product_attention( + query_59, + key_59, + value_59, + attn_mask=attention_mask_20, + dropout_p=0.0, + scale=8.0, + is_causal=False, + ) + query_59 = key_59 = value_59 = attention_mask_20 = None + transpose_39 = attn_output_95.transpose(1, 2) + attn_output_95 = None + attn_output_96 = transpose_39.contiguous() + transpose_39 = None + reshape_59 = attn_output_96.reshape(1, 19, -1) + attn_output_96 = None + attn_output_97 = reshape_59.contiguous() + reshape_59 = None + attn_output_98 = torch._C._nn.linear( + attn_output_97, + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_, + ) + attn_output_97 = l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_ = (None) + attn_output_99 = torch.nn.functional.dropout(attn_output_98, 0.1, False, False) + attn_output_98 = None + hidden_states_195 = attn_output_99 + hidden_states_191 + attn_output_99 = hidden_states_191 = None + hidden_states_196 = torch.nn.functional.layer_norm( + hidden_states_195, + (768,), + l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_ = (None) + hidden_states_197 = torch._C._nn.linear( + hidden_states_196, + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_, + ) + hidden_states_196 = l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_ = (None) + hidden_states_198 = torch._C._nn.gelu(hidden_states_197, approximate="tanh") + hidden_states_197 = None + hidden_states_199 = torch._C._nn.linear( + hidden_states_198, + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_, + l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_, + ) + hidden_states_198 = l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_ = l_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_ = (None) + hidden_states_200 = torch.nn.functional.dropout( + hidden_states_199, 0.1, False, False + ) + hidden_states_199 = None + hidden_states_201 = hidden_states_195 + hidden_states_200 + hidden_states_195 = hidden_states_200 = None + hidden_states_202 = torch.nn.functional.layer_norm( + hidden_states_201, + (768,), + l_self_modules_transformer_modules_ln_f_parameters_weight_, + l_self_modules_transformer_modules_ln_f_parameters_bias_, + 1e-05, + ) + hidden_states_201 = ( + l_self_modules_transformer_modules_ln_f_parameters_weight_ + ) = l_self_modules_transformer_modules_ln_f_parameters_bias_ = None + hidden_states_203 = hidden_states_202.view((1, 19, 768)) + hidden_states_202 = None + lm_logits = torch._C._nn.linear( + hidden_states_203, + l_self_modules_transformer_modules_wte_parameters_weight_, + None, + ) + hidden_states_203 = ( + l_self_modules_transformer_modules_wte_parameters_weight_ + ) = None + return (lm_logits,) diff --git a/samples/transformers-auto-model/tiny_starcoder_py/weight_meta.py b/samples/transformers-auto-model/tiny_starcoder_py/weight_meta.py new file mode 100644 index 0000000000..ee29fff0ff --- /dev/null +++ b/samples/transformers-auto-model/tiny_starcoder_py/weight_meta.py @@ -0,0 +1,2598 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 19] + dtype = "torch.int64" + device = "cpu" + mean = None + std = None + data = [ + 8279, + 30, + 1672, + 636, + 438, + 38873, + 32, + 439, + 3860, + 9608, + 2625, + 7622, + 2898, + 4549, + 461, + 3623, + 46159, + 32, + 225, + ] + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_wte_parameters_weight_: + name = "L_self_modules_transformer_modules_wte_parameters_weight_" + shape = [49152, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 19] + dtype = "torch.int64" + device = "cpu" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_wpe_parameters_weight_: + name = "L_self_modules_transformer_modules_wpe_parameters_weight_" + shape = [8192, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_0_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_0_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_1_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_1_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_2_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_2_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_3_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_3_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_4_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_4_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_5_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_5_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_6_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_6_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_7_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_7_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_8_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_8_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_9_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_: + name = ( + "L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_weight_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_9_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_10_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_10_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_11_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_11_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_12_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_12_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_12_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_13_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_13_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_13_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_14_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_14_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_14_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_15_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_15_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_15_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_16_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_16_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_16_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_17_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_17_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_17_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_18_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_18_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_18_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_19_modules_ln_1_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_weight_" + shape = [896, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_attn_parameters_bias_" + shape = [896] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float16" + device = "cpu" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_: + name = ( + "L_self_modules_transformer_modules_h_modules_19_modules_ln_2_parameters_bias_" + ) + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_transformer_modules_h_modules_19_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_ln_f_parameters_weight_: + name = "L_self_modules_transformer_modules_ln_f_parameters_weight_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 1.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_transformer_modules_ln_f_parameters_bias_: + name = "L_self_modules_transformer_modules_ln_f_parameters_bias_" + shape = [768] + dtype = "torch.float16" + device = "cpu" + mean = 0.000 + std = 0.000 + data = None