From 7bf76b65fd0167f86c7180bb44bf4d02bb1acb0d Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Wed, 6 Aug 2025 10:13:49 +0800 Subject: [PATCH] [New Sample] Add Some NLP Model Computational Graph --- .../albert-base-v2/graph_hash.txt | 1 + .../albert-base-v2/graph_net.json | 5 + .../albert-base-v2/input_meta.py | 0 .../input_tensor_constraints.py | 0 .../albert-base-v2/model.py | 1318 +++++ .../albert-base-v2/weight_meta.py | 803 +++ .../bert-base-uncased/graph_hash.txt | 1 + .../bert-base-uncased/graph_net.json | 5 + .../bert-base-uncased/input_meta.py | 0 .../input_tensor_constraints.py | 0 .../bert-base-uncased/model.py | 1604 ++++++ .../bert-base-uncased/weight_meta.py | 2541 ++++++++++ .../facebook_bart-large-cnn/graph_hash.txt | 1 + .../facebook_bart-large-cnn/graph_net.json | 5 + .../facebook_bart-large-cnn/input_meta.py | 0 .../input_tensor_constraints.py | 0 .../facebook_bart-large-cnn/model.py | 3316 ++++++++++++ .../facebook_bart-large-cnn/weight_meta.py | 3206 ++++++++++++ .../graph_hash.txt | 1 + .../graph_net.json | 5 + .../input_meta.py | 0 .../input_tensor_constraints.py | 0 .../google-bert_bert-large-uncased/model.py | 3080 ++++++++++++ .../weight_meta.py | 4461 +++++++++++++++++ .../graph_hash.txt | 1 + .../graph_net.json | 5 + .../input_meta.py | 0 .../input_tensor_constraints.py | 0 .../model.py | 1705 +++++++ .../weight_meta.py | 2541 ++++++++++ .../gpt2/graph_hash.txt | 1 + .../gpt2/graph_net.json | 5 + .../gpt2/input_meta.py | 0 .../gpt2/input_tensor_constraints.py | 0 samples/transformers-auto-model/gpt2/model.py | 2490 +++++++++ .../gpt2/weight_meta.py | 1498 ++++++ .../roberta-base/graph_hash.txt | 1 + .../roberta-base/graph_net.json | 5 + .../roberta-base/input_meta.py | 0 .../roberta-base/input_tensor_constraints.py | 0 .../roberta-base/model.py | 1618 ++++++ .../roberta-base/weight_meta.py | 2533 ++++++++++ .../t5-small/graph_hash.txt | 1 + .../t5-small/graph_net.json | 5 + .../t5-small/input_meta.py | 0 .../t5-small/input_tensor_constraints.py | 0 .../transformers-auto-model/t5-small/model.py | 1623 ++++++ .../t5-small/weight_meta.py | 828 +++ .../xlnet-base-cased/graph_hash.txt | 1 + .../xlnet-base-cased/graph_net.json | 5 + .../xlnet-base-cased/input_meta.py | 0 .../input_tensor_constraints.py | 0 .../xlnet-base-cased/model.py | 2632 ++++++++++ .../xlnet-base-cased/weight_meta.py | 2150 ++++++++ 54 files changed, 40001 insertions(+) create mode 100644 samples/transformers-auto-model/albert-base-v2/graph_hash.txt create mode 100644 samples/transformers-auto-model/albert-base-v2/graph_net.json create mode 100644 samples/transformers-auto-model/albert-base-v2/input_meta.py create mode 100644 samples/transformers-auto-model/albert-base-v2/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/albert-base-v2/model.py create mode 100644 samples/transformers-auto-model/albert-base-v2/weight_meta.py create mode 100644 samples/transformers-auto-model/bert-base-uncased/graph_hash.txt create mode 100644 samples/transformers-auto-model/bert-base-uncased/graph_net.json create mode 100644 samples/transformers-auto-model/bert-base-uncased/input_meta.py create mode 100644 samples/transformers-auto-model/bert-base-uncased/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/bert-base-uncased/model.py create mode 100644 samples/transformers-auto-model/bert-base-uncased/weight_meta.py create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/graph_hash.txt create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/graph_net.json create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/input_meta.py create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/model.py create mode 100644 samples/transformers-auto-model/facebook_bart-large-cnn/weight_meta.py create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/graph_hash.txt create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/graph_net.json create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/input_meta.py create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/model.py create mode 100644 samples/transformers-auto-model/google-bert_bert-large-uncased/weight_meta.py create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/graph_hash.txt create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/graph_net.json create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/input_meta.py create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/model.py create mode 100644 samples/transformers-auto-model/google_electra-small-discriminator/weight_meta.py create mode 100644 samples/transformers-auto-model/gpt2/graph_hash.txt create mode 100644 samples/transformers-auto-model/gpt2/graph_net.json create mode 100644 samples/transformers-auto-model/gpt2/input_meta.py create mode 100644 samples/transformers-auto-model/gpt2/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/gpt2/model.py create mode 100644 samples/transformers-auto-model/gpt2/weight_meta.py create mode 100644 samples/transformers-auto-model/roberta-base/graph_hash.txt create mode 100644 samples/transformers-auto-model/roberta-base/graph_net.json create mode 100644 samples/transformers-auto-model/roberta-base/input_meta.py create mode 100644 samples/transformers-auto-model/roberta-base/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/roberta-base/model.py create mode 100644 samples/transformers-auto-model/roberta-base/weight_meta.py create mode 100644 samples/transformers-auto-model/t5-small/graph_hash.txt create mode 100644 samples/transformers-auto-model/t5-small/graph_net.json create mode 100644 samples/transformers-auto-model/t5-small/input_meta.py create mode 100644 samples/transformers-auto-model/t5-small/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/t5-small/model.py create mode 100644 samples/transformers-auto-model/t5-small/weight_meta.py create mode 100644 samples/transformers-auto-model/xlnet-base-cased/graph_hash.txt create mode 100644 samples/transformers-auto-model/xlnet-base-cased/graph_net.json create mode 100644 samples/transformers-auto-model/xlnet-base-cased/input_meta.py create mode 100644 samples/transformers-auto-model/xlnet-base-cased/input_tensor_constraints.py create mode 100644 samples/transformers-auto-model/xlnet-base-cased/model.py create mode 100644 samples/transformers-auto-model/xlnet-base-cased/weight_meta.py diff --git a/samples/transformers-auto-model/albert-base-v2/graph_hash.txt b/samples/transformers-auto-model/albert-base-v2/graph_hash.txt new file mode 100644 index 0000000000..1daa2624c6 --- /dev/null +++ b/samples/transformers-auto-model/albert-base-v2/graph_hash.txt @@ -0,0 +1 @@ +63274f127abd3f82bf486c79283364e50c052b227b3c84b4014f8f3bedb06f49 \ No newline at end of file diff --git a/samples/transformers-auto-model/albert-base-v2/graph_net.json b/samples/transformers-auto-model/albert-base-v2/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/albert-base-v2/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/albert-base-v2/input_meta.py b/samples/transformers-auto-model/albert-base-v2/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/albert-base-v2/input_tensor_constraints.py b/samples/transformers-auto-model/albert-base-v2/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/albert-base-v2/model.py b/samples/transformers-auto-model/albert-base-v2/model.py new file mode 100644 index 0000000000..3e541fce49 --- /dev/null +++ b/samples/transformers-auto-model/albert-base-v2/model.py @@ -0,0 +1,1318 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_attention_mask_: torch.Tensor, + L_token_type_ids_: torch.Tensor, + L_self_modules_embeddings_buffers_position_ids_: torch.Tensor, + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_pooler_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_pooler_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_attention_mask_ = L_attention_mask_ + l_token_type_ids_ = L_token_type_ids_ + l_self_modules_embeddings_buffers_position_ids_ = ( + L_self_modules_embeddings_buffers_position_ids_ + ) + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_ + ) + l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_ = L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_ + l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_ = ( + L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_ + ) + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_ + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_ + l_self_modules_pooler_parameters_weight_ = ( + L_self_modules_pooler_parameters_weight_ + ) + l_self_modules_pooler_parameters_bias_ = L_self_modules_pooler_parameters_bias_ + position_ids = l_self_modules_embeddings_buffers_position_ids_[ + (slice(None, None, None), slice(0, 12, None)) + ] + l_self_modules_embeddings_buffers_position_ids_ = None + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_, + 0, + None, + 2.0, + False, + False, + ) + l_input_ids_ = ( + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) = None + token_type_embeddings = torch.nn.functional.embedding( + l_token_type_ids_, + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_token_type_ids_ = ( + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) = None + embeddings = inputs_embeds + token_type_embeddings + inputs_embeds = token_type_embeddings = None + position_embeddings = torch.nn.functional.embedding( + position_ids, + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = ( + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) = None + embeddings += position_embeddings + embeddings_1 = embeddings + embeddings = position_embeddings = None + embeddings_2 = torch.nn.functional.layer_norm( + embeddings_1, + (128,), + l_self_modules_embeddings_modules_layer_norm_parameters_weight_, + l_self_modules_embeddings_modules_layer_norm_parameters_bias_, + 1e-12, + ) + embeddings_1 = ( + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ + ) = l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = None + embeddings_3 = torch.nn.functional.dropout(embeddings_2, 0, False, False) + embeddings_2 = None + getitem_1 = l_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_attention_mask_ = None + expand = getitem_1.expand(1, 1, 12, 12) + getitem_1 = None + expanded_mask = expand.to(torch.float32) + expand = None + tensor = torch.tensor(1.0, dtype=torch.float32) + inverted_mask = tensor - expanded_mask + tensor = expanded_mask = None + to_1 = inverted_mask.to(torch.bool) + extended_attention_mask = inverted_mask.masked_fill( + to_1, -3.4028234663852886e38 + ) + inverted_mask = to_1 = None + hidden_states = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_, + l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_, + ) + embeddings_3 = l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_ = ( + l_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_ + ) = None + linear_1 = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view = linear_1.view(1, -1, 12, 64) + linear_1 = None + query_layer = view.transpose(1, 2) + view = None + linear_2 = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_1 = linear_2.view(1, -1, 12, 64) + linear_2 = None + key_layer = view_1.transpose(1, 2) + view_1 = None + linear_3 = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_2 = linear_3.view(1, -1, 12, 64) + linear_3 = None + value_layer = view_2.transpose(1, 2) + view_2 = None + attention_output = torch._C._nn.scaled_dot_product_attention( + query=query_layer, + key=key_layer, + value=value_layer, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer = key_layer = value_layer = None + attention_output_1 = attention_output.transpose(1, 2) + attention_output = None + attention_output_2 = attention_output_1.reshape(1, 12, 768) + attention_output_1 = None + projected_context_layer = torch._C._nn.linear( + attention_output_2, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_2 = None + projected_context_layer_dropout = torch.nn.functional.dropout( + projected_context_layer, 0, False, False + ) + projected_context_layer = None + add_1 = hidden_states + projected_context_layer_dropout + hidden_states = projected_context_layer_dropout = None + layernormed_context_layer = torch.nn.functional.layer_norm( + add_1, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_1 = None + ffn_output = torch._C._nn.linear( + layernormed_context_layer, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul = 0.5 * ffn_output + pow_1 = torch.pow(ffn_output, 3.0) + mul_1 = 0.044715 * pow_1 + pow_1 = None + add_2 = ffn_output + mul_1 + ffn_output = mul_1 = None + mul_2 = 0.7978845608028654 * add_2 + add_2 = None + tanh = torch.tanh(mul_2) + mul_2 = None + add_3 = 1.0 + tanh + tanh = None + ffn_output_1 = mul * add_3 + mul = add_3 = None + ffn_output_2 = torch._C._nn.linear( + ffn_output_1, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_1 = None + add_4 = ffn_output_2 + layernormed_context_layer + ffn_output_2 = layernormed_context_layer = None + hidden_states_1 = torch.nn.functional.layer_norm( + add_4, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_4 = None + linear_7 = torch._C._nn.linear( + hidden_states_1, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_3 = linear_7.view(1, -1, 12, 64) + linear_7 = None + query_layer_1 = view_3.transpose(1, 2) + view_3 = None + linear_8 = torch._C._nn.linear( + hidden_states_1, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_4 = linear_8.view(1, -1, 12, 64) + linear_8 = None + key_layer_1 = view_4.transpose(1, 2) + view_4 = None + linear_9 = torch._C._nn.linear( + hidden_states_1, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_5 = linear_9.view(1, -1, 12, 64) + linear_9 = None + value_layer_1 = view_5.transpose(1, 2) + view_5 = None + attention_output_3 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_1, + key=key_layer_1, + value=value_layer_1, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_1 = key_layer_1 = value_layer_1 = None + attention_output_4 = attention_output_3.transpose(1, 2) + attention_output_3 = None + attention_output_5 = attention_output_4.reshape(1, 12, 768) + attention_output_4 = None + projected_context_layer_1 = torch._C._nn.linear( + attention_output_5, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_5 = None + projected_context_layer_dropout_1 = torch.nn.functional.dropout( + projected_context_layer_1, 0, False, False + ) + projected_context_layer_1 = None + add_5 = hidden_states_1 + projected_context_layer_dropout_1 + hidden_states_1 = projected_context_layer_dropout_1 = None + layernormed_context_layer_1 = torch.nn.functional.layer_norm( + add_5, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_5 = None + ffn_output_3 = torch._C._nn.linear( + layernormed_context_layer_1, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_4 = 0.5 * ffn_output_3 + pow_2 = torch.pow(ffn_output_3, 3.0) + mul_5 = 0.044715 * pow_2 + pow_2 = None + add_6 = ffn_output_3 + mul_5 + ffn_output_3 = mul_5 = None + mul_6 = 0.7978845608028654 * add_6 + add_6 = None + tanh_1 = torch.tanh(mul_6) + mul_6 = None + add_7 = 1.0 + tanh_1 + tanh_1 = None + ffn_output_4 = mul_4 * add_7 + mul_4 = add_7 = None + ffn_output_5 = torch._C._nn.linear( + ffn_output_4, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_4 = None + add_8 = ffn_output_5 + layernormed_context_layer_1 + ffn_output_5 = layernormed_context_layer_1 = None + hidden_states_2 = torch.nn.functional.layer_norm( + add_8, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_8 = None + linear_13 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_6 = linear_13.view(1, -1, 12, 64) + linear_13 = None + query_layer_2 = view_6.transpose(1, 2) + view_6 = None + linear_14 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_7 = linear_14.view(1, -1, 12, 64) + linear_14 = None + key_layer_2 = view_7.transpose(1, 2) + view_7 = None + linear_15 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_8 = linear_15.view(1, -1, 12, 64) + linear_15 = None + value_layer_2 = view_8.transpose(1, 2) + view_8 = None + attention_output_6 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_2, + key=key_layer_2, + value=value_layer_2, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_2 = key_layer_2 = value_layer_2 = None + attention_output_7 = attention_output_6.transpose(1, 2) + attention_output_6 = None + attention_output_8 = attention_output_7.reshape(1, 12, 768) + attention_output_7 = None + projected_context_layer_2 = torch._C._nn.linear( + attention_output_8, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_8 = None + projected_context_layer_dropout_2 = torch.nn.functional.dropout( + projected_context_layer_2, 0, False, False + ) + projected_context_layer_2 = None + add_9 = hidden_states_2 + projected_context_layer_dropout_2 + hidden_states_2 = projected_context_layer_dropout_2 = None + layernormed_context_layer_2 = torch.nn.functional.layer_norm( + add_9, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_9 = None + ffn_output_6 = torch._C._nn.linear( + layernormed_context_layer_2, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_8 = 0.5 * ffn_output_6 + pow_3 = torch.pow(ffn_output_6, 3.0) + mul_9 = 0.044715 * pow_3 + pow_3 = None + add_10 = ffn_output_6 + mul_9 + ffn_output_6 = mul_9 = None + mul_10 = 0.7978845608028654 * add_10 + add_10 = None + tanh_2 = torch.tanh(mul_10) + mul_10 = None + add_11 = 1.0 + tanh_2 + tanh_2 = None + ffn_output_7 = mul_8 * add_11 + mul_8 = add_11 = None + ffn_output_8 = torch._C._nn.linear( + ffn_output_7, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_7 = None + add_12 = ffn_output_8 + layernormed_context_layer_2 + ffn_output_8 = layernormed_context_layer_2 = None + hidden_states_3 = torch.nn.functional.layer_norm( + add_12, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_12 = None + linear_19 = torch._C._nn.linear( + hidden_states_3, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_9 = linear_19.view(1, -1, 12, 64) + linear_19 = None + query_layer_3 = view_9.transpose(1, 2) + view_9 = None + linear_20 = torch._C._nn.linear( + hidden_states_3, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_10 = linear_20.view(1, -1, 12, 64) + linear_20 = None + key_layer_3 = view_10.transpose(1, 2) + view_10 = None + linear_21 = torch._C._nn.linear( + hidden_states_3, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_11 = linear_21.view(1, -1, 12, 64) + linear_21 = None + value_layer_3 = view_11.transpose(1, 2) + view_11 = None + attention_output_9 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_3, + key=key_layer_3, + value=value_layer_3, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_3 = key_layer_3 = value_layer_3 = None + attention_output_10 = attention_output_9.transpose(1, 2) + attention_output_9 = None + attention_output_11 = attention_output_10.reshape(1, 12, 768) + attention_output_10 = None + projected_context_layer_3 = torch._C._nn.linear( + attention_output_11, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_11 = None + projected_context_layer_dropout_3 = torch.nn.functional.dropout( + projected_context_layer_3, 0, False, False + ) + projected_context_layer_3 = None + add_13 = hidden_states_3 + projected_context_layer_dropout_3 + hidden_states_3 = projected_context_layer_dropout_3 = None + layernormed_context_layer_3 = torch.nn.functional.layer_norm( + add_13, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_13 = None + ffn_output_9 = torch._C._nn.linear( + layernormed_context_layer_3, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_12 = 0.5 * ffn_output_9 + pow_4 = torch.pow(ffn_output_9, 3.0) + mul_13 = 0.044715 * pow_4 + pow_4 = None + add_14 = ffn_output_9 + mul_13 + ffn_output_9 = mul_13 = None + mul_14 = 0.7978845608028654 * add_14 + add_14 = None + tanh_3 = torch.tanh(mul_14) + mul_14 = None + add_15 = 1.0 + tanh_3 + tanh_3 = None + ffn_output_10 = mul_12 * add_15 + mul_12 = add_15 = None + ffn_output_11 = torch._C._nn.linear( + ffn_output_10, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_10 = None + add_16 = ffn_output_11 + layernormed_context_layer_3 + ffn_output_11 = layernormed_context_layer_3 = None + hidden_states_4 = torch.nn.functional.layer_norm( + add_16, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_16 = None + linear_25 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_12 = linear_25.view(1, -1, 12, 64) + linear_25 = None + query_layer_4 = view_12.transpose(1, 2) + view_12 = None + linear_26 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_13 = linear_26.view(1, -1, 12, 64) + linear_26 = None + key_layer_4 = view_13.transpose(1, 2) + view_13 = None + linear_27 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_14 = linear_27.view(1, -1, 12, 64) + linear_27 = None + value_layer_4 = view_14.transpose(1, 2) + view_14 = None + attention_output_12 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_4, + key=key_layer_4, + value=value_layer_4, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_4 = key_layer_4 = value_layer_4 = None + attention_output_13 = attention_output_12.transpose(1, 2) + attention_output_12 = None + attention_output_14 = attention_output_13.reshape(1, 12, 768) + attention_output_13 = None + projected_context_layer_4 = torch._C._nn.linear( + attention_output_14, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_14 = None + projected_context_layer_dropout_4 = torch.nn.functional.dropout( + projected_context_layer_4, 0, False, False + ) + projected_context_layer_4 = None + add_17 = hidden_states_4 + projected_context_layer_dropout_4 + hidden_states_4 = projected_context_layer_dropout_4 = None + layernormed_context_layer_4 = torch.nn.functional.layer_norm( + add_17, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_17 = None + ffn_output_12 = torch._C._nn.linear( + layernormed_context_layer_4, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_16 = 0.5 * ffn_output_12 + pow_5 = torch.pow(ffn_output_12, 3.0) + mul_17 = 0.044715 * pow_5 + pow_5 = None + add_18 = ffn_output_12 + mul_17 + ffn_output_12 = mul_17 = None + mul_18 = 0.7978845608028654 * add_18 + add_18 = None + tanh_4 = torch.tanh(mul_18) + mul_18 = None + add_19 = 1.0 + tanh_4 + tanh_4 = None + ffn_output_13 = mul_16 * add_19 + mul_16 = add_19 = None + ffn_output_14 = torch._C._nn.linear( + ffn_output_13, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_13 = None + add_20 = ffn_output_14 + layernormed_context_layer_4 + ffn_output_14 = layernormed_context_layer_4 = None + hidden_states_5 = torch.nn.functional.layer_norm( + add_20, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_20 = None + linear_31 = torch._C._nn.linear( + hidden_states_5, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_15 = linear_31.view(1, -1, 12, 64) + linear_31 = None + query_layer_5 = view_15.transpose(1, 2) + view_15 = None + linear_32 = torch._C._nn.linear( + hidden_states_5, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_16 = linear_32.view(1, -1, 12, 64) + linear_32 = None + key_layer_5 = view_16.transpose(1, 2) + view_16 = None + linear_33 = torch._C._nn.linear( + hidden_states_5, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_17 = linear_33.view(1, -1, 12, 64) + linear_33 = None + value_layer_5 = view_17.transpose(1, 2) + view_17 = None + attention_output_15 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_5, + key=key_layer_5, + value=value_layer_5, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_5 = key_layer_5 = value_layer_5 = None + attention_output_16 = attention_output_15.transpose(1, 2) + attention_output_15 = None + attention_output_17 = attention_output_16.reshape(1, 12, 768) + attention_output_16 = None + projected_context_layer_5 = torch._C._nn.linear( + attention_output_17, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_17 = None + projected_context_layer_dropout_5 = torch.nn.functional.dropout( + projected_context_layer_5, 0, False, False + ) + projected_context_layer_5 = None + add_21 = hidden_states_5 + projected_context_layer_dropout_5 + hidden_states_5 = projected_context_layer_dropout_5 = None + layernormed_context_layer_5 = torch.nn.functional.layer_norm( + add_21, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_21 = None + ffn_output_15 = torch._C._nn.linear( + layernormed_context_layer_5, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_20 = 0.5 * ffn_output_15 + pow_6 = torch.pow(ffn_output_15, 3.0) + mul_21 = 0.044715 * pow_6 + pow_6 = None + add_22 = ffn_output_15 + mul_21 + ffn_output_15 = mul_21 = None + mul_22 = 0.7978845608028654 * add_22 + add_22 = None + tanh_5 = torch.tanh(mul_22) + mul_22 = None + add_23 = 1.0 + tanh_5 + tanh_5 = None + ffn_output_16 = mul_20 * add_23 + mul_20 = add_23 = None + ffn_output_17 = torch._C._nn.linear( + ffn_output_16, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_16 = None + add_24 = ffn_output_17 + layernormed_context_layer_5 + ffn_output_17 = layernormed_context_layer_5 = None + hidden_states_6 = torch.nn.functional.layer_norm( + add_24, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_24 = None + linear_37 = torch._C._nn.linear( + hidden_states_6, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_18 = linear_37.view(1, -1, 12, 64) + linear_37 = None + query_layer_6 = view_18.transpose(1, 2) + view_18 = None + linear_38 = torch._C._nn.linear( + hidden_states_6, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_19 = linear_38.view(1, -1, 12, 64) + linear_38 = None + key_layer_6 = view_19.transpose(1, 2) + view_19 = None + linear_39 = torch._C._nn.linear( + hidden_states_6, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_20 = linear_39.view(1, -1, 12, 64) + linear_39 = None + value_layer_6 = view_20.transpose(1, 2) + view_20 = None + attention_output_18 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_6, + key=key_layer_6, + value=value_layer_6, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_6 = key_layer_6 = value_layer_6 = None + attention_output_19 = attention_output_18.transpose(1, 2) + attention_output_18 = None + attention_output_20 = attention_output_19.reshape(1, 12, 768) + attention_output_19 = None + projected_context_layer_6 = torch._C._nn.linear( + attention_output_20, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_20 = None + projected_context_layer_dropout_6 = torch.nn.functional.dropout( + projected_context_layer_6, 0, False, False + ) + projected_context_layer_6 = None + add_25 = hidden_states_6 + projected_context_layer_dropout_6 + hidden_states_6 = projected_context_layer_dropout_6 = None + layernormed_context_layer_6 = torch.nn.functional.layer_norm( + add_25, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_25 = None + ffn_output_18 = torch._C._nn.linear( + layernormed_context_layer_6, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_24 = 0.5 * ffn_output_18 + pow_7 = torch.pow(ffn_output_18, 3.0) + mul_25 = 0.044715 * pow_7 + pow_7 = None + add_26 = ffn_output_18 + mul_25 + ffn_output_18 = mul_25 = None + mul_26 = 0.7978845608028654 * add_26 + add_26 = None + tanh_6 = torch.tanh(mul_26) + mul_26 = None + add_27 = 1.0 + tanh_6 + tanh_6 = None + ffn_output_19 = mul_24 * add_27 + mul_24 = add_27 = None + ffn_output_20 = torch._C._nn.linear( + ffn_output_19, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_19 = None + add_28 = ffn_output_20 + layernormed_context_layer_6 + ffn_output_20 = layernormed_context_layer_6 = None + hidden_states_7 = torch.nn.functional.layer_norm( + add_28, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_28 = None + linear_43 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_21 = linear_43.view(1, -1, 12, 64) + linear_43 = None + query_layer_7 = view_21.transpose(1, 2) + view_21 = None + linear_44 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_22 = linear_44.view(1, -1, 12, 64) + linear_44 = None + key_layer_7 = view_22.transpose(1, 2) + view_22 = None + linear_45 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_23 = linear_45.view(1, -1, 12, 64) + linear_45 = None + value_layer_7 = view_23.transpose(1, 2) + view_23 = None + attention_output_21 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_7, + key=key_layer_7, + value=value_layer_7, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_7 = key_layer_7 = value_layer_7 = None + attention_output_22 = attention_output_21.transpose(1, 2) + attention_output_21 = None + attention_output_23 = attention_output_22.reshape(1, 12, 768) + attention_output_22 = None + projected_context_layer_7 = torch._C._nn.linear( + attention_output_23, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_23 = None + projected_context_layer_dropout_7 = torch.nn.functional.dropout( + projected_context_layer_7, 0, False, False + ) + projected_context_layer_7 = None + add_29 = hidden_states_7 + projected_context_layer_dropout_7 + hidden_states_7 = projected_context_layer_dropout_7 = None + layernormed_context_layer_7 = torch.nn.functional.layer_norm( + add_29, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_29 = None + ffn_output_21 = torch._C._nn.linear( + layernormed_context_layer_7, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_28 = 0.5 * ffn_output_21 + pow_8 = torch.pow(ffn_output_21, 3.0) + mul_29 = 0.044715 * pow_8 + pow_8 = None + add_30 = ffn_output_21 + mul_29 + ffn_output_21 = mul_29 = None + mul_30 = 0.7978845608028654 * add_30 + add_30 = None + tanh_7 = torch.tanh(mul_30) + mul_30 = None + add_31 = 1.0 + tanh_7 + tanh_7 = None + ffn_output_22 = mul_28 * add_31 + mul_28 = add_31 = None + ffn_output_23 = torch._C._nn.linear( + ffn_output_22, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_22 = None + add_32 = ffn_output_23 + layernormed_context_layer_7 + ffn_output_23 = layernormed_context_layer_7 = None + hidden_states_8 = torch.nn.functional.layer_norm( + add_32, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_32 = None + linear_49 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_24 = linear_49.view(1, -1, 12, 64) + linear_49 = None + query_layer_8 = view_24.transpose(1, 2) + view_24 = None + linear_50 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_25 = linear_50.view(1, -1, 12, 64) + linear_50 = None + key_layer_8 = view_25.transpose(1, 2) + view_25 = None + linear_51 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_26 = linear_51.view(1, -1, 12, 64) + linear_51 = None + value_layer_8 = view_26.transpose(1, 2) + view_26 = None + attention_output_24 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_8, + key=key_layer_8, + value=value_layer_8, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_8 = key_layer_8 = value_layer_8 = None + attention_output_25 = attention_output_24.transpose(1, 2) + attention_output_24 = None + attention_output_26 = attention_output_25.reshape(1, 12, 768) + attention_output_25 = None + projected_context_layer_8 = torch._C._nn.linear( + attention_output_26, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_26 = None + projected_context_layer_dropout_8 = torch.nn.functional.dropout( + projected_context_layer_8, 0, False, False + ) + projected_context_layer_8 = None + add_33 = hidden_states_8 + projected_context_layer_dropout_8 + hidden_states_8 = projected_context_layer_dropout_8 = None + layernormed_context_layer_8 = torch.nn.functional.layer_norm( + add_33, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_33 = None + ffn_output_24 = torch._C._nn.linear( + layernormed_context_layer_8, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_32 = 0.5 * ffn_output_24 + pow_9 = torch.pow(ffn_output_24, 3.0) + mul_33 = 0.044715 * pow_9 + pow_9 = None + add_34 = ffn_output_24 + mul_33 + ffn_output_24 = mul_33 = None + mul_34 = 0.7978845608028654 * add_34 + add_34 = None + tanh_8 = torch.tanh(mul_34) + mul_34 = None + add_35 = 1.0 + tanh_8 + tanh_8 = None + ffn_output_25 = mul_32 * add_35 + mul_32 = add_35 = None + ffn_output_26 = torch._C._nn.linear( + ffn_output_25, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_25 = None + add_36 = ffn_output_26 + layernormed_context_layer_8 + ffn_output_26 = layernormed_context_layer_8 = None + hidden_states_9 = torch.nn.functional.layer_norm( + add_36, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_36 = None + linear_55 = torch._C._nn.linear( + hidden_states_9, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_27 = linear_55.view(1, -1, 12, 64) + linear_55 = None + query_layer_9 = view_27.transpose(1, 2) + view_27 = None + linear_56 = torch._C._nn.linear( + hidden_states_9, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_28 = linear_56.view(1, -1, 12, 64) + linear_56 = None + key_layer_9 = view_28.transpose(1, 2) + view_28 = None + linear_57 = torch._C._nn.linear( + hidden_states_9, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_29 = linear_57.view(1, -1, 12, 64) + linear_57 = None + value_layer_9 = view_29.transpose(1, 2) + view_29 = None + attention_output_27 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_9, + key=key_layer_9, + value=value_layer_9, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_9 = key_layer_9 = value_layer_9 = None + attention_output_28 = attention_output_27.transpose(1, 2) + attention_output_27 = None + attention_output_29 = attention_output_28.reshape(1, 12, 768) + attention_output_28 = None + projected_context_layer_9 = torch._C._nn.linear( + attention_output_29, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_29 = None + projected_context_layer_dropout_9 = torch.nn.functional.dropout( + projected_context_layer_9, 0, False, False + ) + projected_context_layer_9 = None + add_37 = hidden_states_9 + projected_context_layer_dropout_9 + hidden_states_9 = projected_context_layer_dropout_9 = None + layernormed_context_layer_9 = torch.nn.functional.layer_norm( + add_37, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_37 = None + ffn_output_27 = torch._C._nn.linear( + layernormed_context_layer_9, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_36 = 0.5 * ffn_output_27 + pow_10 = torch.pow(ffn_output_27, 3.0) + mul_37 = 0.044715 * pow_10 + pow_10 = None + add_38 = ffn_output_27 + mul_37 + ffn_output_27 = mul_37 = None + mul_38 = 0.7978845608028654 * add_38 + add_38 = None + tanh_9 = torch.tanh(mul_38) + mul_38 = None + add_39 = 1.0 + tanh_9 + tanh_9 = None + ffn_output_28 = mul_36 * add_39 + mul_36 = add_39 = None + ffn_output_29 = torch._C._nn.linear( + ffn_output_28, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_28 = None + add_40 = ffn_output_29 + layernormed_context_layer_9 + ffn_output_29 = layernormed_context_layer_9 = None + hidden_states_10 = torch.nn.functional.layer_norm( + add_40, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_40 = None + linear_61 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + view_30 = linear_61.view(1, -1, 12, 64) + linear_61 = None + query_layer_10 = view_30.transpose(1, 2) + view_30 = None + linear_62 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + view_31 = linear_62.view(1, -1, 12, 64) + linear_62 = None + key_layer_10 = view_31.transpose(1, 2) + view_31 = None + linear_63 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + view_32 = linear_63.view(1, -1, 12, 64) + linear_63 = None + value_layer_10 = view_32.transpose(1, 2) + view_32 = None + attention_output_30 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_10, + key=key_layer_10, + value=value_layer_10, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_10 = key_layer_10 = value_layer_10 = None + attention_output_31 = attention_output_30.transpose(1, 2) + attention_output_30 = None + attention_output_32 = attention_output_31.reshape(1, 12, 768) + attention_output_31 = None + projected_context_layer_10 = torch._C._nn.linear( + attention_output_32, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_32 = None + projected_context_layer_dropout_10 = torch.nn.functional.dropout( + projected_context_layer_10, 0, False, False + ) + projected_context_layer_10 = None + add_41 = hidden_states_10 + projected_context_layer_dropout_10 + hidden_states_10 = projected_context_layer_dropout_10 = None + layernormed_context_layer_10 = torch.nn.functional.layer_norm( + add_41, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_41 = None + ffn_output_30 = torch._C._nn.linear( + layernormed_context_layer_10, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + mul_40 = 0.5 * ffn_output_30 + pow_11 = torch.pow(ffn_output_30, 3.0) + mul_41 = 0.044715 * pow_11 + pow_11 = None + add_42 = ffn_output_30 + mul_41 + ffn_output_30 = mul_41 = None + mul_42 = 0.7978845608028654 * add_42 + add_42 = None + tanh_10 = torch.tanh(mul_42) + mul_42 = None + add_43 = 1.0 + tanh_10 + tanh_10 = None + ffn_output_31 = mul_40 * add_43 + mul_40 = add_43 = None + ffn_output_32 = torch._C._nn.linear( + ffn_output_31, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_31 = None + add_44 = ffn_output_32 + layernormed_context_layer_10 + ffn_output_32 = layernormed_context_layer_10 = None + hidden_states_11 = torch.nn.functional.layer_norm( + add_44, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_44 = None + linear_67 = torch._C._nn.linear( + hidden_states_11, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_ = (None) + view_33 = linear_67.view(1, -1, 12, 64) + linear_67 = None + query_layer_11 = view_33.transpose(1, 2) + view_33 = None + linear_68 = torch._C._nn.linear( + hidden_states_11, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_ = (None) + view_34 = linear_68.view(1, -1, 12, 64) + linear_68 = None + key_layer_11 = view_34.transpose(1, 2) + view_34 = None + linear_69 = torch._C._nn.linear( + hidden_states_11, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_ = (None) + view_35 = linear_69.view(1, -1, 12, 64) + linear_69 = None + value_layer_11 = view_35.transpose(1, 2) + view_35 = None + attention_output_33 = torch._C._nn.scaled_dot_product_attention( + query=query_layer_11, + key=key_layer_11, + value=value_layer_11, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_11 = key_layer_11 = value_layer_11 = extended_attention_mask = None + attention_output_34 = attention_output_33.transpose(1, 2) + attention_output_33 = None + attention_output_35 = attention_output_34.reshape(1, 12, 768) + attention_output_34 = None + projected_context_layer_11 = torch._C._nn.linear( + attention_output_35, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_, + ) + attention_output_35 = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_ = (None) + projected_context_layer_dropout_11 = torch.nn.functional.dropout( + projected_context_layer_11, 0, False, False + ) + projected_context_layer_11 = None + add_45 = hidden_states_11 + projected_context_layer_dropout_11 + hidden_states_11 = projected_context_layer_dropout_11 = None + layernormed_context_layer_11 = torch.nn.functional.layer_norm( + add_45, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_45 = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_layer_norm_parameters_bias_ = (None) + ffn_output_33 = torch._C._nn.linear( + layernormed_context_layer_11, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_, + ) + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_ = (None) + mul_44 = 0.5 * ffn_output_33 + pow_12 = torch.pow(ffn_output_33, 3.0) + mul_45 = 0.044715 * pow_12 + pow_12 = None + add_46 = ffn_output_33 + mul_45 + ffn_output_33 = mul_45 = None + mul_46 = 0.7978845608028654 * add_46 + add_46 = None + tanh_11 = torch.tanh(mul_46) + mul_46 = None + add_47 = 1.0 + tanh_11 + tanh_11 = None + ffn_output_34 = mul_44 * add_47 + mul_44 = add_47 = None + ffn_output_35 = torch._C._nn.linear( + ffn_output_34, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_, + ) + ffn_output_34 = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_ = (None) + add_48 = ffn_output_35 + layernormed_context_layer_11 + ffn_output_35 = layernormed_context_layer_11 = None + hidden_states_12 = torch.nn.functional.layer_norm( + add_48, + (768,), + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_, + 1e-12, + ) + add_48 = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_ = (None) + getitem_2 = hidden_states_12[(slice(None, None, None), 0)] + linear_73 = torch._C._nn.linear( + getitem_2, + l_self_modules_pooler_parameters_weight_, + l_self_modules_pooler_parameters_bias_, + ) + getitem_2 = ( + l_self_modules_pooler_parameters_weight_ + ) = l_self_modules_pooler_parameters_bias_ = None + pooled_output = torch.tanh(linear_73) + linear_73 = None + return (hidden_states_12, pooled_output) diff --git a/samples/transformers-auto-model/albert-base-v2/weight_meta.py b/samples/transformers-auto-model/albert-base-v2/weight_meta.py new file mode 100644 index 0000000000..b0863c53cb --- /dev/null +++ b/samples/transformers-auto-model/albert-base-v2/weight_meta.py @@ -0,0 +1,803 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 12] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [2, 7210, 2328, 25, 147, 20, 10962, 16439, 7210, 18, 9, 3] + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 12] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_token_type_ids_: + name = "L_token_type_ids_" + shape = [1, 12] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_buffers_position_ids_: + name = "L_self_modules_embeddings_buffers_position_ids_" + shape = [1, 512] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + ] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_word_embeddings_parameters_weight_" + shape = [30000, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_" + shape = [2, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_position_embeddings_parameters_weight_" + shape = [512, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_weight_" + shape = [128] + dtype = "torch.float32" + device = "cuda:0" + mean = 3.264 + std = 0.281 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_bias_" + shape = [128] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.035 + std = 0.618 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_: + name = ( + "L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_weight_" + ) + shape = [768, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_: + name = "L_self_modules_encoder_modules_embedding_hidden_mapping_in_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.023 + std = 0.206 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.061 + std = 0.838 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.013 + std = 0.567 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.547 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.570 + std = 0.231 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_attention_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.255 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.556 + std = 0.478 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_ffn_output_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.014 + std = 0.588 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.350 + std = 0.134 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_: + name = "L_self_modules_encoder_modules_albert_layer_groups_modules_0_modules_albert_layers_modules_0_modules_full_layer_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.041 + std = 0.235 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_parameters_weight_: + name = "L_self_modules_pooler_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_parameters_bias_: + name = "L_self_modules_pooler_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.529 + data = None diff --git a/samples/transformers-auto-model/bert-base-uncased/graph_hash.txt b/samples/transformers-auto-model/bert-base-uncased/graph_hash.txt new file mode 100644 index 0000000000..896709c041 --- /dev/null +++ b/samples/transformers-auto-model/bert-base-uncased/graph_hash.txt @@ -0,0 +1 @@ +183ca2c0c1206a1210783ec1a1b9b94595cd2650e0f4b9ec19452dff6c7c80e4 \ No newline at end of file diff --git a/samples/transformers-auto-model/bert-base-uncased/graph_net.json b/samples/transformers-auto-model/bert-base-uncased/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/bert-base-uncased/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/bert-base-uncased/input_meta.py b/samples/transformers-auto-model/bert-base-uncased/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/bert-base-uncased/input_tensor_constraints.py b/samples/transformers-auto-model/bert-base-uncased/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/bert-base-uncased/model.py b/samples/transformers-auto-model/bert-base-uncased/model.py new file mode 100644 index 0000000000..f5e601c666 --- /dev/null +++ b/samples/transformers-auto-model/bert-base-uncased/model.py @@ -0,0 +1,1604 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_token_type_ids_: torch.Tensor, + L_self_modules_embeddings_buffers_position_ids_: torch.Tensor, + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_attention_mask_: torch.Tensor, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_token_type_ids_ = L_token_type_ids_ + l_self_modules_embeddings_buffers_position_ids_ = ( + L_self_modules_embeddings_buffers_position_ids_ + ) + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_ + ) + l_attention_mask_ = L_attention_mask_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_pooler_modules_dense_parameters_weight_ = ( + L_self_modules_pooler_modules_dense_parameters_weight_ + ) + l_self_modules_pooler_modules_dense_parameters_bias_ = ( + L_self_modules_pooler_modules_dense_parameters_bias_ + ) + position_ids = l_self_modules_embeddings_buffers_position_ids_[ + (slice(None, None, None), slice(0, 11, None)) + ] + l_self_modules_embeddings_buffers_position_ids_ = None + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_, + 0, + None, + 2.0, + False, + False, + ) + l_input_ids_ = ( + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) = None + token_type_embeddings = torch.nn.functional.embedding( + l_token_type_ids_, + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_token_type_ids_ = ( + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) = None + embeddings = inputs_embeds + token_type_embeddings + inputs_embeds = token_type_embeddings = None + position_embeddings = torch.nn.functional.embedding( + position_ids, + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = ( + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) = None + embeddings += position_embeddings + embeddings_1 = embeddings + embeddings = position_embeddings = None + embeddings_2 = torch.nn.functional.layer_norm( + embeddings_1, + (768,), + l_self_modules_embeddings_modules_layer_norm_parameters_weight_, + l_self_modules_embeddings_modules_layer_norm_parameters_bias_, + 1e-12, + ) + embeddings_1 = ( + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ + ) = l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = None + embeddings_3 = torch.nn.functional.dropout(embeddings_2, 0.1, False, False) + embeddings_2 = None + getitem_1 = l_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_attention_mask_ = None + expand = getitem_1.expand(1, 1, 11, 11) + getitem_1 = None + expanded_mask = expand.to(torch.float32) + expand = None + tensor = torch.tensor(1.0, dtype=torch.float32) + inverted_mask = tensor - expanded_mask + tensor = expanded_mask = None + to_1 = inverted_mask.to(torch.bool) + extended_attention_mask = inverted_mask.masked_fill( + to_1, -3.4028234663852886e38 + ) + inverted_mask = to_1 = None + linear = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view = linear.view(1, -1, 12, 64) + linear = None + query_layer = view.transpose(1, 2) + view = None + linear_1 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_1 = linear_1.view(1, -1, 12, 64) + linear_1 = None + key_layer = view_1.transpose(1, 2) + view_1 = None + linear_2 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_2 = linear_2.view(1, -1, 12, 64) + linear_2 = None + value_layer = view_2.transpose(1, 2) + view_2 = None + attn_output = torch._C._nn.scaled_dot_product_attention( + query_layer, + key_layer, + value_layer, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer = key_layer = value_layer = None + attn_output_1 = attn_output.transpose(1, 2) + attn_output = None + attn_output_2 = attn_output_1.reshape(1, 11, 768) + attn_output_1 = None + hidden_states = torch._C._nn.linear( + attn_output_2, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_2 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_1 = torch.nn.functional.dropout(hidden_states, 0.1, False, False) + hidden_states = None + add_1 = hidden_states_1 + embeddings_3 + hidden_states_1 = embeddings_3 = None + hidden_states_2 = torch.nn.functional.layer_norm( + add_1, + (768,), + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_1 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_3 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_4 = torch._C._nn.gelu(hidden_states_3) + hidden_states_3 = None + hidden_states_5 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_4 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_6 = torch.nn.functional.dropout( + hidden_states_5, 0.1, False, False + ) + hidden_states_5 = None + add_2 = hidden_states_6 + hidden_states_2 + hidden_states_6 = hidden_states_2 = None + hidden_states_7 = torch.nn.functional.layer_norm( + add_2, + (768,), + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_2 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_6 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_3 = linear_6.view(1, -1, 12, 64) + linear_6 = None + query_layer_1 = view_3.transpose(1, 2) + view_3 = None + linear_7 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_4 = linear_7.view(1, -1, 12, 64) + linear_7 = None + key_layer_1 = view_4.transpose(1, 2) + view_4 = None + linear_8 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_5 = linear_8.view(1, -1, 12, 64) + linear_8 = None + value_layer_1 = view_5.transpose(1, 2) + view_5 = None + attn_output_3 = torch._C._nn.scaled_dot_product_attention( + query_layer_1, + key_layer_1, + value_layer_1, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_1 = key_layer_1 = value_layer_1 = None + attn_output_4 = attn_output_3.transpose(1, 2) + attn_output_3 = None + attn_output_5 = attn_output_4.reshape(1, 11, 768) + attn_output_4 = None + hidden_states_8 = torch._C._nn.linear( + attn_output_5, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_9 = torch.nn.functional.dropout( + hidden_states_8, 0.1, False, False + ) + hidden_states_8 = None + add_3 = hidden_states_9 + hidden_states_7 + hidden_states_9 = hidden_states_7 = None + hidden_states_10 = torch.nn.functional.layer_norm( + add_3, + (768,), + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_3 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_11 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_12 = torch._C._nn.gelu(hidden_states_11) + hidden_states_11 = None + hidden_states_13 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_12 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_14 = torch.nn.functional.dropout( + hidden_states_13, 0.1, False, False + ) + hidden_states_13 = None + add_4 = hidden_states_14 + hidden_states_10 + hidden_states_14 = hidden_states_10 = None + hidden_states_15 = torch.nn.functional.layer_norm( + add_4, + (768,), + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_4 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_12 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_6 = linear_12.view(1, -1, 12, 64) + linear_12 = None + query_layer_2 = view_6.transpose(1, 2) + view_6 = None + linear_13 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_7 = linear_13.view(1, -1, 12, 64) + linear_13 = None + key_layer_2 = view_7.transpose(1, 2) + view_7 = None + linear_14 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_8 = linear_14.view(1, -1, 12, 64) + linear_14 = None + value_layer_2 = view_8.transpose(1, 2) + view_8 = None + attn_output_6 = torch._C._nn.scaled_dot_product_attention( + query_layer_2, + key_layer_2, + value_layer_2, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_2 = key_layer_2 = value_layer_2 = None + attn_output_7 = attn_output_6.transpose(1, 2) + attn_output_6 = None + attn_output_8 = attn_output_7.reshape(1, 11, 768) + attn_output_7 = None + hidden_states_16 = torch._C._nn.linear( + attn_output_8, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_8 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_17 = torch.nn.functional.dropout( + hidden_states_16, 0.1, False, False + ) + hidden_states_16 = None + add_5 = hidden_states_17 + hidden_states_15 + hidden_states_17 = hidden_states_15 = None + hidden_states_18 = torch.nn.functional.layer_norm( + add_5, + (768,), + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_5 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_19 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_20 = torch._C._nn.gelu(hidden_states_19) + hidden_states_19 = None + hidden_states_21 = torch._C._nn.linear( + hidden_states_20, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_20 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_22 = torch.nn.functional.dropout( + hidden_states_21, 0.1, False, False + ) + hidden_states_21 = None + add_6 = hidden_states_22 + hidden_states_18 + hidden_states_22 = hidden_states_18 = None + hidden_states_23 = torch.nn.functional.layer_norm( + add_6, + (768,), + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_6 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_18 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_9 = linear_18.view(1, -1, 12, 64) + linear_18 = None + query_layer_3 = view_9.transpose(1, 2) + view_9 = None + linear_19 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_10 = linear_19.view(1, -1, 12, 64) + linear_19 = None + key_layer_3 = view_10.transpose(1, 2) + view_10 = None + linear_20 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_11 = linear_20.view(1, -1, 12, 64) + linear_20 = None + value_layer_3 = view_11.transpose(1, 2) + view_11 = None + attn_output_9 = torch._C._nn.scaled_dot_product_attention( + query_layer_3, + key_layer_3, + value_layer_3, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_3 = key_layer_3 = value_layer_3 = None + attn_output_10 = attn_output_9.transpose(1, 2) + attn_output_9 = None + attn_output_11 = attn_output_10.reshape(1, 11, 768) + attn_output_10 = None + hidden_states_24 = torch._C._nn.linear( + attn_output_11, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_11 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_25 = torch.nn.functional.dropout( + hidden_states_24, 0.1, False, False + ) + hidden_states_24 = None + add_7 = hidden_states_25 + hidden_states_23 + hidden_states_25 = hidden_states_23 = None + hidden_states_26 = torch.nn.functional.layer_norm( + add_7, + (768,), + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_7 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_27 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_28 = torch._C._nn.gelu(hidden_states_27) + hidden_states_27 = None + hidden_states_29 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_28 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_30 = torch.nn.functional.dropout( + hidden_states_29, 0.1, False, False + ) + hidden_states_29 = None + add_8 = hidden_states_30 + hidden_states_26 + hidden_states_30 = hidden_states_26 = None + hidden_states_31 = torch.nn.functional.layer_norm( + add_8, + (768,), + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_8 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_24 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_12 = linear_24.view(1, -1, 12, 64) + linear_24 = None + query_layer_4 = view_12.transpose(1, 2) + view_12 = None + linear_25 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_13 = linear_25.view(1, -1, 12, 64) + linear_25 = None + key_layer_4 = view_13.transpose(1, 2) + view_13 = None + linear_26 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_14 = linear_26.view(1, -1, 12, 64) + linear_26 = None + value_layer_4 = view_14.transpose(1, 2) + view_14 = None + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_layer_4, + key_layer_4, + value_layer_4, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_4 = key_layer_4 = value_layer_4 = None + attn_output_13 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_14 = attn_output_13.reshape(1, 11, 768) + attn_output_13 = None + hidden_states_32 = torch._C._nn.linear( + attn_output_14, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_14 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_33 = torch.nn.functional.dropout( + hidden_states_32, 0.1, False, False + ) + hidden_states_32 = None + add_9 = hidden_states_33 + hidden_states_31 + hidden_states_33 = hidden_states_31 = None + hidden_states_34 = torch.nn.functional.layer_norm( + add_9, + (768,), + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_9 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_35 = torch._C._nn.linear( + hidden_states_34, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_36 = torch._C._nn.gelu(hidden_states_35) + hidden_states_35 = None + hidden_states_37 = torch._C._nn.linear( + hidden_states_36, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_36 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_38 = torch.nn.functional.dropout( + hidden_states_37, 0.1, False, False + ) + hidden_states_37 = None + add_10 = hidden_states_38 + hidden_states_34 + hidden_states_38 = hidden_states_34 = None + hidden_states_39 = torch.nn.functional.layer_norm( + add_10, + (768,), + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_10 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_30 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_15 = linear_30.view(1, -1, 12, 64) + linear_30 = None + query_layer_5 = view_15.transpose(1, 2) + view_15 = None + linear_31 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_16 = linear_31.view(1, -1, 12, 64) + linear_31 = None + key_layer_5 = view_16.transpose(1, 2) + view_16 = None + linear_32 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_17 = linear_32.view(1, -1, 12, 64) + linear_32 = None + value_layer_5 = view_17.transpose(1, 2) + view_17 = None + attn_output_15 = torch._C._nn.scaled_dot_product_attention( + query_layer_5, + key_layer_5, + value_layer_5, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_5 = key_layer_5 = value_layer_5 = None + attn_output_16 = attn_output_15.transpose(1, 2) + attn_output_15 = None + attn_output_17 = attn_output_16.reshape(1, 11, 768) + attn_output_16 = None + hidden_states_40 = torch._C._nn.linear( + attn_output_17, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_17 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_41 = torch.nn.functional.dropout( + hidden_states_40, 0.1, False, False + ) + hidden_states_40 = None + add_11 = hidden_states_41 + hidden_states_39 + hidden_states_41 = hidden_states_39 = None + hidden_states_42 = torch.nn.functional.layer_norm( + add_11, + (768,), + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_11 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_43 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_44 = torch._C._nn.gelu(hidden_states_43) + hidden_states_43 = None + hidden_states_45 = torch._C._nn.linear( + hidden_states_44, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_44 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_46 = torch.nn.functional.dropout( + hidden_states_45, 0.1, False, False + ) + hidden_states_45 = None + add_12 = hidden_states_46 + hidden_states_42 + hidden_states_46 = hidden_states_42 = None + hidden_states_47 = torch.nn.functional.layer_norm( + add_12, + (768,), + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_12 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_36 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_18 = linear_36.view(1, -1, 12, 64) + linear_36 = None + query_layer_6 = view_18.transpose(1, 2) + view_18 = None + linear_37 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_19 = linear_37.view(1, -1, 12, 64) + linear_37 = None + key_layer_6 = view_19.transpose(1, 2) + view_19 = None + linear_38 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_20 = linear_38.view(1, -1, 12, 64) + linear_38 = None + value_layer_6 = view_20.transpose(1, 2) + view_20 = None + attn_output_18 = torch._C._nn.scaled_dot_product_attention( + query_layer_6, + key_layer_6, + value_layer_6, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_6 = key_layer_6 = value_layer_6 = None + attn_output_19 = attn_output_18.transpose(1, 2) + attn_output_18 = None + attn_output_20 = attn_output_19.reshape(1, 11, 768) + attn_output_19 = None + hidden_states_48 = torch._C._nn.linear( + attn_output_20, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_20 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_49 = torch.nn.functional.dropout( + hidden_states_48, 0.1, False, False + ) + hidden_states_48 = None + add_13 = hidden_states_49 + hidden_states_47 + hidden_states_49 = hidden_states_47 = None + hidden_states_50 = torch.nn.functional.layer_norm( + add_13, + (768,), + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_13 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_51 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_52 = torch._C._nn.gelu(hidden_states_51) + hidden_states_51 = None + hidden_states_53 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_52 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_54 = torch.nn.functional.dropout( + hidden_states_53, 0.1, False, False + ) + hidden_states_53 = None + add_14 = hidden_states_54 + hidden_states_50 + hidden_states_54 = hidden_states_50 = None + hidden_states_55 = torch.nn.functional.layer_norm( + add_14, + (768,), + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_14 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_42 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_21 = linear_42.view(1, -1, 12, 64) + linear_42 = None + query_layer_7 = view_21.transpose(1, 2) + view_21 = None + linear_43 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_22 = linear_43.view(1, -1, 12, 64) + linear_43 = None + key_layer_7 = view_22.transpose(1, 2) + view_22 = None + linear_44 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_23 = linear_44.view(1, -1, 12, 64) + linear_44 = None + value_layer_7 = view_23.transpose(1, 2) + view_23 = None + attn_output_21 = torch._C._nn.scaled_dot_product_attention( + query_layer_7, + key_layer_7, + value_layer_7, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_7 = key_layer_7 = value_layer_7 = None + attn_output_22 = attn_output_21.transpose(1, 2) + attn_output_21 = None + attn_output_23 = attn_output_22.reshape(1, 11, 768) + attn_output_22 = None + hidden_states_56 = torch._C._nn.linear( + attn_output_23, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_23 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_57 = torch.nn.functional.dropout( + hidden_states_56, 0.1, False, False + ) + hidden_states_56 = None + add_15 = hidden_states_57 + hidden_states_55 + hidden_states_57 = hidden_states_55 = None + hidden_states_58 = torch.nn.functional.layer_norm( + add_15, + (768,), + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_15 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_59 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_60 = torch._C._nn.gelu(hidden_states_59) + hidden_states_59 = None + hidden_states_61 = torch._C._nn.linear( + hidden_states_60, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_60 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_62 = torch.nn.functional.dropout( + hidden_states_61, 0.1, False, False + ) + hidden_states_61 = None + add_16 = hidden_states_62 + hidden_states_58 + hidden_states_62 = hidden_states_58 = None + hidden_states_63 = torch.nn.functional.layer_norm( + add_16, + (768,), + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_16 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_48 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_24 = linear_48.view(1, -1, 12, 64) + linear_48 = None + query_layer_8 = view_24.transpose(1, 2) + view_24 = None + linear_49 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_25 = linear_49.view(1, -1, 12, 64) + linear_49 = None + key_layer_8 = view_25.transpose(1, 2) + view_25 = None + linear_50 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_26 = linear_50.view(1, -1, 12, 64) + linear_50 = None + value_layer_8 = view_26.transpose(1, 2) + view_26 = None + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_layer_8, + key_layer_8, + value_layer_8, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_8 = key_layer_8 = value_layer_8 = None + attn_output_25 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_26 = attn_output_25.reshape(1, 11, 768) + attn_output_25 = None + hidden_states_64 = torch._C._nn.linear( + attn_output_26, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_26 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_65 = torch.nn.functional.dropout( + hidden_states_64, 0.1, False, False + ) + hidden_states_64 = None + add_17 = hidden_states_65 + hidden_states_63 + hidden_states_65 = hidden_states_63 = None + hidden_states_66 = torch.nn.functional.layer_norm( + add_17, + (768,), + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_17 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_67 = torch._C._nn.linear( + hidden_states_66, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_68 = torch._C._nn.gelu(hidden_states_67) + hidden_states_67 = None + hidden_states_69 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_68 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_70 = torch.nn.functional.dropout( + hidden_states_69, 0.1, False, False + ) + hidden_states_69 = None + add_18 = hidden_states_70 + hidden_states_66 + hidden_states_70 = hidden_states_66 = None + hidden_states_71 = torch.nn.functional.layer_norm( + add_18, + (768,), + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_18 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_54 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_27 = linear_54.view(1, -1, 12, 64) + linear_54 = None + query_layer_9 = view_27.transpose(1, 2) + view_27 = None + linear_55 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_28 = linear_55.view(1, -1, 12, 64) + linear_55 = None + key_layer_9 = view_28.transpose(1, 2) + view_28 = None + linear_56 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_29 = linear_56.view(1, -1, 12, 64) + linear_56 = None + value_layer_9 = view_29.transpose(1, 2) + view_29 = None + attn_output_27 = torch._C._nn.scaled_dot_product_attention( + query_layer_9, + key_layer_9, + value_layer_9, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_9 = key_layer_9 = value_layer_9 = None + attn_output_28 = attn_output_27.transpose(1, 2) + attn_output_27 = None + attn_output_29 = attn_output_28.reshape(1, 11, 768) + attn_output_28 = None + hidden_states_72 = torch._C._nn.linear( + attn_output_29, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_29 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_73 = torch.nn.functional.dropout( + hidden_states_72, 0.1, False, False + ) + hidden_states_72 = None + add_19 = hidden_states_73 + hidden_states_71 + hidden_states_73 = hidden_states_71 = None + hidden_states_74 = torch.nn.functional.layer_norm( + add_19, + (768,), + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_19 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_75 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_76 = torch._C._nn.gelu(hidden_states_75) + hidden_states_75 = None + hidden_states_77 = torch._C._nn.linear( + hidden_states_76, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_76 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_78 = torch.nn.functional.dropout( + hidden_states_77, 0.1, False, False + ) + hidden_states_77 = None + add_20 = hidden_states_78 + hidden_states_74 + hidden_states_78 = hidden_states_74 = None + hidden_states_79 = torch.nn.functional.layer_norm( + add_20, + (768,), + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_20 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_60 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_30 = linear_60.view(1, -1, 12, 64) + linear_60 = None + query_layer_10 = view_30.transpose(1, 2) + view_30 = None + linear_61 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_31 = linear_61.view(1, -1, 12, 64) + linear_61 = None + key_layer_10 = view_31.transpose(1, 2) + view_31 = None + linear_62 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_32 = linear_62.view(1, -1, 12, 64) + linear_62 = None + value_layer_10 = view_32.transpose(1, 2) + view_32 = None + attn_output_30 = torch._C._nn.scaled_dot_product_attention( + query_layer_10, + key_layer_10, + value_layer_10, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_10 = key_layer_10 = value_layer_10 = None + attn_output_31 = attn_output_30.transpose(1, 2) + attn_output_30 = None + attn_output_32 = attn_output_31.reshape(1, 11, 768) + attn_output_31 = None + hidden_states_80 = torch._C._nn.linear( + attn_output_32, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_32 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_81 = torch.nn.functional.dropout( + hidden_states_80, 0.1, False, False + ) + hidden_states_80 = None + add_21 = hidden_states_81 + hidden_states_79 + hidden_states_81 = hidden_states_79 = None + hidden_states_82 = torch.nn.functional.layer_norm( + add_21, + (768,), + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_21 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_83 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_84 = torch._C._nn.gelu(hidden_states_83) + hidden_states_83 = None + hidden_states_85 = torch._C._nn.linear( + hidden_states_84, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_84 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_86 = torch.nn.functional.dropout( + hidden_states_85, 0.1, False, False + ) + hidden_states_85 = None + add_22 = hidden_states_86 + hidden_states_82 + hidden_states_86 = hidden_states_82 = None + hidden_states_87 = torch.nn.functional.layer_norm( + add_22, + (768,), + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_22 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_66 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_33 = linear_66.view(1, -1, 12, 64) + linear_66 = None + query_layer_11 = view_33.transpose(1, 2) + view_33 = None + linear_67 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_34 = linear_67.view(1, -1, 12, 64) + linear_67 = None + key_layer_11 = view_34.transpose(1, 2) + view_34 = None + linear_68 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_35 = linear_68.view(1, -1, 12, 64) + linear_68 = None + value_layer_11 = view_35.transpose(1, 2) + view_35 = None + attn_output_33 = torch._C._nn.scaled_dot_product_attention( + query_layer_11, + key_layer_11, + value_layer_11, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_11 = key_layer_11 = value_layer_11 = extended_attention_mask = None + attn_output_34 = attn_output_33.transpose(1, 2) + attn_output_33 = None + attn_output_35 = attn_output_34.reshape(1, 11, 768) + attn_output_34 = None + hidden_states_88 = torch._C._nn.linear( + attn_output_35, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_35 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_89 = torch.nn.functional.dropout( + hidden_states_88, 0.1, False, False + ) + hidden_states_88 = None + add_23 = hidden_states_89 + hidden_states_87 + hidden_states_89 = hidden_states_87 = None + hidden_states_90 = torch.nn.functional.layer_norm( + add_23, + (768,), + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_23 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_91 = torch._C._nn.linear( + hidden_states_90, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_92 = torch._C._nn.gelu(hidden_states_91) + hidden_states_91 = None + hidden_states_93 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_92 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_94 = torch.nn.functional.dropout( + hidden_states_93, 0.1, False, False + ) + hidden_states_93 = None + add_24 = hidden_states_94 + hidden_states_90 + hidden_states_94 = hidden_states_90 = None + hidden_states_95 = torch.nn.functional.layer_norm( + add_24, + (768,), + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_24 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = (None) + first_token_tensor = hidden_states_95[(slice(None, None, None), 0)] + pooled_output = torch._C._nn.linear( + first_token_tensor, + l_self_modules_pooler_modules_dense_parameters_weight_, + l_self_modules_pooler_modules_dense_parameters_bias_, + ) + first_token_tensor = ( + l_self_modules_pooler_modules_dense_parameters_weight_ + ) = l_self_modules_pooler_modules_dense_parameters_bias_ = None + pooled_output_1 = torch.tanh(pooled_output) + pooled_output = None + return (hidden_states_95, pooled_output_1) diff --git a/samples/transformers-auto-model/bert-base-uncased/weight_meta.py b/samples/transformers-auto-model/bert-base-uncased/weight_meta.py new file mode 100644 index 0000000000..1f00429697 --- /dev/null +++ b/samples/transformers-auto-model/bert-base-uncased/weight_meta.py @@ -0,0 +1,2541 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [101, 10629, 7159, 2003, 2109, 2000, 14817, 15078, 19287, 1012, 102] + + +class Program_weight_tensor_meta_L_token_type_ids_: + name = "L_token_type_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_buffers_position_ids_: + name = "L_self_modules_embeddings_buffers_position_ids_" + shape = [1, 512] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + ] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_word_embeddings_parameters_weight_" + shape = [30522, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.028 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_" + shape = [2, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_position_embeddings_parameters_weight_" + shape = [512, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.016 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.849 + std = 0.134 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.020 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.281 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.958 + std = 0.083 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.018 + std = 0.329 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.108 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.756 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.037 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.016 + std = 0.181 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.876 + std = 0.088 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.018 + std = 0.194 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.870 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.034 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.868 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.015 + std = 0.176 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.096 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.851 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.031 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.137 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.863 + std = 0.118 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.160 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.097 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.811 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.033 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.148 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.839 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.018 + std = 0.144 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.097 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.840 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.031 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.126 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.006 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.847 + std = 0.111 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.018 + std = 0.137 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.098 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.832 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.032 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.828 + std = 0.108 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.019 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.095 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.834 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.033 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.822 + std = 0.106 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.025 + std = 0.144 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.102 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.810 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.033 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.174 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.007 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.837 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.027 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.100 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.831 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.035 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.192 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.007 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.817 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.030 + std = 0.133 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.105 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.801 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.032 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.230 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.006 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.841 + std = 0.132 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.021 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.104 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.817 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.040 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.219 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.853 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.037 + std = 0.108 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.076 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.633 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.019 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_weight_: + name = "L_self_modules_pooler_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_bias_: + name = "L_self_modules_pooler_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.035 + data = None diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/graph_hash.txt b/samples/transformers-auto-model/facebook_bart-large-cnn/graph_hash.txt new file mode 100644 index 0000000000..7939418348 --- /dev/null +++ b/samples/transformers-auto-model/facebook_bart-large-cnn/graph_hash.txt @@ -0,0 +1 @@ +d3ff94c0b4b32a3c536266d706537da8638bb28dc648384e773cd007b1947506 \ No newline at end of file diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/graph_net.json b/samples/transformers-auto-model/facebook_bart-large-cnn/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/facebook_bart-large-cnn/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/input_meta.py b/samples/transformers-auto-model/facebook_bart-large-cnn/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/input_tensor_constraints.py b/samples/transformers-auto-model/facebook_bart-large-cnn/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/model.py b/samples/transformers-auto-model/facebook_bart-large-cnn/model.py new file mode 100644 index 0000000000..60064a4607 --- /dev/null +++ b/samples/transformers-auto-model/facebook_bart-large-cnn/model.py @@ -0,0 +1,3316 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_inputs_embeds_: torch.Tensor, + L_encoder_hidden_states_: torch.Tensor, + L_encoder_attention_mask_: torch.Tensor, + L_self_modules_embed_positions_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layernorm_embedding_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layernorm_embedding_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_fc1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_fc1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_fc2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_fc2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_inputs_embeds_ = L_inputs_embeds_ + l_encoder_hidden_states_ = L_encoder_hidden_states_ + l_encoder_attention_mask_ = L_encoder_attention_mask_ + l_self_modules_embed_positions_parameters_weight_ = ( + L_self_modules_embed_positions_parameters_weight_ + ) + l_self_modules_layernorm_embedding_parameters_weight_ = ( + L_self_modules_layernorm_embedding_parameters_weight_ + ) + l_self_modules_layernorm_embedding_parameters_bias_ = ( + L_self_modules_layernorm_embedding_parameters_bias_ + ) + l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_0_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_0_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_0_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_0_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_0_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_0_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_0_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_0_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_1_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_1_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_1_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_1_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_1_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_1_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_1_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_1_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_2_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_2_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_2_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_2_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_2_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_2_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_2_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_2_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_3_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_3_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_3_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_3_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_3_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_3_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_3_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_3_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_4_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_4_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_4_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_4_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_4_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_4_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_4_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_4_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_5_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_5_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_5_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_5_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_5_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_5_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_5_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_5_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_6_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_6_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_6_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_6_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_6_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_6_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_6_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_6_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_7_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_7_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_7_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_7_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_7_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_7_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_7_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_7_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_8_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_8_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_8_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_8_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_8_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_8_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_8_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_8_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_9_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_9_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_9_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_9_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_9_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_9_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_9_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_9_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_10_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_10_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_10_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_10_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_10_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_10_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_10_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_10_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_ + ) + l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_ + l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_ = L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_ + l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_ = L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_ + l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_ = L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_ + l_self_modules_layers_modules_11_modules_fc1_parameters_weight_ = ( + L_self_modules_layers_modules_11_modules_fc1_parameters_weight_ + ) + l_self_modules_layers_modules_11_modules_fc1_parameters_bias_ = ( + L_self_modules_layers_modules_11_modules_fc1_parameters_bias_ + ) + l_self_modules_layers_modules_11_modules_fc2_parameters_weight_ = ( + L_self_modules_layers_modules_11_modules_fc2_parameters_weight_ + ) + l_self_modules_layers_modules_11_modules_fc2_parameters_bias_ = ( + L_self_modules_layers_modules_11_modules_fc2_parameters_bias_ + ) + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_ + ) + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_ = ( + L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_ + ) + cache_position = torch.arange(0, 11, device=device(type="cuda", index=0)) + causal_mask = torch.full( + (11, 12), + fill_value=-3.4028234663852886e38, + dtype=torch.float32, + device=device(type="cuda", index=0), + ) + causal_mask_1 = torch.triu(causal_mask, diagonal=1) + causal_mask = None + arange_1 = torch.arange(12, device=device(type="cuda", index=0)) + reshape = cache_position.reshape(-1, 1) + gt = arange_1 > reshape + arange_1 = reshape = None + causal_mask_1 *= gt + causal_mask_2 = causal_mask_1 + causal_mask_1 = gt = None + getitem = causal_mask_2[ + (None, None, slice(None, None, None), slice(None, None, None)) + ] + causal_mask_2 = None + causal_mask_3 = getitem.expand(1, 1, -1, -1) + getitem = None + getitem_1 = l_encoder_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_encoder_attention_mask_ = None + expand_1 = getitem_1.expand(1, 1, 11, 11) + getitem_1 = None + expanded_mask = expand_1.to(torch.float32) + expand_1 = None + tensor = torch.tensor(1.0, dtype=torch.float32) + inverted_mask = tensor - expanded_mask + tensor = expanded_mask = None + to_1 = inverted_mask.to(torch.bool) + encoder_attention_mask = inverted_mask.masked_fill(to_1, -3.4028234663852886e38) + inverted_mask = to_1 = None + position_ids = cache_position.unsqueeze(0) + cache_position = None + add = position_ids + 2 + position_ids = None + positions = torch.nn.functional.embedding( + add, + l_self_modules_embed_positions_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + add = l_self_modules_embed_positions_parameters_weight_ = None + positions_1 = positions.to(device(type="cuda", index=0)) + positions = None + hidden_states = l_inputs_embeds_ + positions_1 + l_inputs_embeds_ = positions_1 = None + hidden_states_1 = torch.nn.functional.layer_norm( + hidden_states, + (1024,), + l_self_modules_layernorm_embedding_parameters_weight_, + l_self_modules_layernorm_embedding_parameters_bias_, + 1e-05, + ) + hidden_states = ( + l_self_modules_layernorm_embedding_parameters_weight_ + ) = l_self_modules_layernorm_embedding_parameters_bias_ = None + hidden_states_2 = torch.nn.functional.dropout( + hidden_states_1, p=0.1, training=False + ) + hidden_states_1 = None + linear = torch._C._nn.linear( + hidden_states_2, + l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view = linear.view(1, 11, -1, 64) + linear = None + query_states = view.transpose(1, 2) + view = None + key_states = torch._C._nn.linear( + hidden_states_2, + l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states = torch._C._nn.linear( + hidden_states_2, + l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_1 = key_states.view(1, 11, -1, 64) + key_states = None + key_states_1 = view_1.transpose(1, 2) + view_1 = None + view_2 = value_states.view(1, 11, -1, 64) + value_states = None + value_states_1 = view_2.transpose(1, 2) + view_2 = None + attention_mask = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query = query_states.contiguous() + query_states = None + key = key_states_1.contiguous() + value = value_states_1.contiguous() + attn_output = torch._C._nn.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query = key = value = attention_mask = None + transpose_3 = attn_output.transpose(1, 2) + attn_output = None + attn_output_1 = transpose_3.contiguous() + transpose_3 = None + reshape_1 = attn_output_1.reshape(1, 11, -1) + attn_output_1 = None + attn_output_2 = reshape_1.contiguous() + reshape_1 = None + attn_output_3 = torch._C._nn.linear( + attn_output_2, + l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_2 = l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_3 = torch.nn.functional.dropout( + attn_output_3, p=0.1, training=False + ) + attn_output_3 = None + hidden_states_4 = hidden_states_2 + hidden_states_3 + hidden_states_2 = hidden_states_3 = None + hidden_states_5 = torch.nn.functional.layer_norm( + hidden_states_4, + (1024,), + l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_4 = l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_4 = torch._C._nn.linear( + hidden_states_5, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_3 = linear_4.view(1, 11, -1, 64) + linear_4 = None + query_states_1 = view_3.transpose(1, 2) + view_3 = None + key_states_2 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_2 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_4 = key_states_2.view(1, 11, -1, 64) + key_states_2 = None + key_states_3 = view_4.transpose(1, 2) + view_4 = None + view_5 = value_states_2.view(1, 11, -1, 64) + value_states_2 = None + value_states_3 = view_5.transpose(1, 2) + view_5 = None + attention_mask_1 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_1 = query_states_1.contiguous() + query_states_1 = None + key_1 = key_states_3.contiguous() + value_1 = value_states_3.contiguous() + attn_output_4 = torch._C._nn.scaled_dot_product_attention( + query_1, + key_1, + value_1, + attn_mask=attention_mask_1, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_1 = key_1 = value_1 = attention_mask_1 = None + transpose_7 = attn_output_4.transpose(1, 2) + attn_output_4 = None + attn_output_5 = transpose_7.contiguous() + transpose_7 = None + reshape_2 = attn_output_5.reshape(1, 11, -1) + attn_output_5 = None + attn_output_6 = reshape_2.contiguous() + reshape_2 = None + attn_output_7 = torch._C._nn.linear( + attn_output_6, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_6 = l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_6 = torch.nn.functional.dropout( + attn_output_7, p=0.1, training=False + ) + attn_output_7 = None + hidden_states_7 = hidden_states_5 + hidden_states_6 + hidden_states_5 = hidden_states_6 = None + hidden_states_8 = torch.nn.functional.layer_norm( + hidden_states_7, + (1024,), + l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_7 = l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_8 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_layers_modules_0_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_0_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_0_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_0_modules_fc1_parameters_bias_ + ) = None + hidden_states_9 = torch._C._nn.gelu(linear_8) + linear_8 = None + hidden_states_10 = torch.nn.functional.dropout( + hidden_states_9, p=0.0, training=False + ) + hidden_states_9 = None + hidden_states_11 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_layers_modules_0_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_0_modules_fc2_parameters_bias_, + ) + hidden_states_10 = ( + l_self_modules_layers_modules_0_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_0_modules_fc2_parameters_bias_ = None + hidden_states_12 = torch.nn.functional.dropout( + hidden_states_11, p=0.1, training=False + ) + hidden_states_11 = None + hidden_states_13 = hidden_states_8 + hidden_states_12 + hidden_states_8 = hidden_states_12 = None + hidden_states_14 = torch.nn.functional.layer_norm( + hidden_states_13, + (1024,), + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_13 = ( + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_ + ) = None + linear_10 = torch._C._nn.linear( + hidden_states_14, + l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_6 = linear_10.view(1, 11, -1, 64) + linear_10 = None + query_states_2 = view_6.transpose(1, 2) + view_6 = None + key_states_4 = torch._C._nn.linear( + hidden_states_14, + l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_4 = torch._C._nn.linear( + hidden_states_14, + l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_7 = key_states_4.view(1, 11, -1, 64) + key_states_4 = None + key_states_5 = view_7.transpose(1, 2) + view_7 = None + view_8 = value_states_4.view(1, 11, -1, 64) + value_states_4 = None + value_states_5 = view_8.transpose(1, 2) + view_8 = None + attention_mask_2 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_2 = query_states_2.contiguous() + query_states_2 = None + key_2 = key_states_5.contiguous() + value_2 = value_states_5.contiguous() + attn_output_8 = torch._C._nn.scaled_dot_product_attention( + query_2, + key_2, + value_2, + attn_mask=attention_mask_2, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_2 = key_2 = value_2 = attention_mask_2 = None + transpose_11 = attn_output_8.transpose(1, 2) + attn_output_8 = None + attn_output_9 = transpose_11.contiguous() + transpose_11 = None + reshape_3 = attn_output_9.reshape(1, 11, -1) + attn_output_9 = None + attn_output_10 = reshape_3.contiguous() + reshape_3 = None + attn_output_11 = torch._C._nn.linear( + attn_output_10, + l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_10 = l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_15 = torch.nn.functional.dropout( + attn_output_11, p=0.1, training=False + ) + attn_output_11 = None + hidden_states_16 = hidden_states_14 + hidden_states_15 + hidden_states_14 = hidden_states_15 = None + hidden_states_17 = torch.nn.functional.layer_norm( + hidden_states_16, + (1024,), + l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_16 = l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_14 = torch._C._nn.linear( + hidden_states_17, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_9 = linear_14.view(1, 11, -1, 64) + linear_14 = None + query_states_3 = view_9.transpose(1, 2) + view_9 = None + key_states_6 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_6 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_10 = key_states_6.view(1, 11, -1, 64) + key_states_6 = None + key_states_7 = view_10.transpose(1, 2) + view_10 = None + view_11 = value_states_6.view(1, 11, -1, 64) + value_states_6 = None + value_states_7 = view_11.transpose(1, 2) + view_11 = None + attention_mask_3 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_3 = query_states_3.contiguous() + query_states_3 = None + key_3 = key_states_7.contiguous() + value_3 = value_states_7.contiguous() + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_3, + key_3, + value_3, + attn_mask=attention_mask_3, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_3 = key_3 = value_3 = attention_mask_3 = None + transpose_15 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_13 = transpose_15.contiguous() + transpose_15 = None + reshape_4 = attn_output_13.reshape(1, 11, -1) + attn_output_13 = None + attn_output_14 = reshape_4.contiguous() + reshape_4 = None + attn_output_15 = torch._C._nn.linear( + attn_output_14, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_14 = l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_18 = torch.nn.functional.dropout( + attn_output_15, p=0.1, training=False + ) + attn_output_15 = None + hidden_states_19 = hidden_states_17 + hidden_states_18 + hidden_states_17 = hidden_states_18 = None + hidden_states_20 = torch.nn.functional.layer_norm( + hidden_states_19, + (1024,), + l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_19 = l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_18 = torch._C._nn.linear( + hidden_states_20, + l_self_modules_layers_modules_1_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_1_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_1_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_1_modules_fc1_parameters_bias_ + ) = None + hidden_states_21 = torch._C._nn.gelu(linear_18) + linear_18 = None + hidden_states_22 = torch.nn.functional.dropout( + hidden_states_21, p=0.0, training=False + ) + hidden_states_21 = None + hidden_states_23 = torch._C._nn.linear( + hidden_states_22, + l_self_modules_layers_modules_1_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_1_modules_fc2_parameters_bias_, + ) + hidden_states_22 = ( + l_self_modules_layers_modules_1_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_1_modules_fc2_parameters_bias_ = None + hidden_states_24 = torch.nn.functional.dropout( + hidden_states_23, p=0.1, training=False + ) + hidden_states_23 = None + hidden_states_25 = hidden_states_20 + hidden_states_24 + hidden_states_20 = hidden_states_24 = None + hidden_states_26 = torch.nn.functional.layer_norm( + hidden_states_25, + (1024,), + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_25 = ( + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_ + ) = None + linear_20 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_12 = linear_20.view(1, 11, -1, 64) + linear_20 = None + query_states_4 = view_12.transpose(1, 2) + view_12 = None + key_states_8 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_8 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_13 = key_states_8.view(1, 11, -1, 64) + key_states_8 = None + key_states_9 = view_13.transpose(1, 2) + view_13 = None + view_14 = value_states_8.view(1, 11, -1, 64) + value_states_8 = None + value_states_9 = view_14.transpose(1, 2) + view_14 = None + attention_mask_4 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_4 = query_states_4.contiguous() + query_states_4 = None + key_4 = key_states_9.contiguous() + value_4 = value_states_9.contiguous() + attn_output_16 = torch._C._nn.scaled_dot_product_attention( + query_4, + key_4, + value_4, + attn_mask=attention_mask_4, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_4 = key_4 = value_4 = attention_mask_4 = None + transpose_19 = attn_output_16.transpose(1, 2) + attn_output_16 = None + attn_output_17 = transpose_19.contiguous() + transpose_19 = None + reshape_5 = attn_output_17.reshape(1, 11, -1) + attn_output_17 = None + attn_output_18 = reshape_5.contiguous() + reshape_5 = None + attn_output_19 = torch._C._nn.linear( + attn_output_18, + l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_18 = l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_27 = torch.nn.functional.dropout( + attn_output_19, p=0.1, training=False + ) + attn_output_19 = None + hidden_states_28 = hidden_states_26 + hidden_states_27 + hidden_states_26 = hidden_states_27 = None + hidden_states_29 = torch.nn.functional.layer_norm( + hidden_states_28, + (1024,), + l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_28 = l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_24 = torch._C._nn.linear( + hidden_states_29, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_15 = linear_24.view(1, 11, -1, 64) + linear_24 = None + query_states_5 = view_15.transpose(1, 2) + view_15 = None + key_states_10 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_10 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_16 = key_states_10.view(1, 11, -1, 64) + key_states_10 = None + key_states_11 = view_16.transpose(1, 2) + view_16 = None + view_17 = value_states_10.view(1, 11, -1, 64) + value_states_10 = None + value_states_11 = view_17.transpose(1, 2) + view_17 = None + attention_mask_5 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_5 = query_states_5.contiguous() + query_states_5 = None + key_5 = key_states_11.contiguous() + value_5 = value_states_11.contiguous() + attn_output_20 = torch._C._nn.scaled_dot_product_attention( + query_5, + key_5, + value_5, + attn_mask=attention_mask_5, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_5 = key_5 = value_5 = attention_mask_5 = None + transpose_23 = attn_output_20.transpose(1, 2) + attn_output_20 = None + attn_output_21 = transpose_23.contiguous() + transpose_23 = None + reshape_6 = attn_output_21.reshape(1, 11, -1) + attn_output_21 = None + attn_output_22 = reshape_6.contiguous() + reshape_6 = None + attn_output_23 = torch._C._nn.linear( + attn_output_22, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_22 = l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_30 = torch.nn.functional.dropout( + attn_output_23, p=0.1, training=False + ) + attn_output_23 = None + hidden_states_31 = hidden_states_29 + hidden_states_30 + hidden_states_29 = hidden_states_30 = None + hidden_states_32 = torch.nn.functional.layer_norm( + hidden_states_31, + (1024,), + l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_31 = l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_28 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_layers_modules_2_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_2_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_2_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_2_modules_fc1_parameters_bias_ + ) = None + hidden_states_33 = torch._C._nn.gelu(linear_28) + linear_28 = None + hidden_states_34 = torch.nn.functional.dropout( + hidden_states_33, p=0.0, training=False + ) + hidden_states_33 = None + hidden_states_35 = torch._C._nn.linear( + hidden_states_34, + l_self_modules_layers_modules_2_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_2_modules_fc2_parameters_bias_, + ) + hidden_states_34 = ( + l_self_modules_layers_modules_2_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_2_modules_fc2_parameters_bias_ = None + hidden_states_36 = torch.nn.functional.dropout( + hidden_states_35, p=0.1, training=False + ) + hidden_states_35 = None + hidden_states_37 = hidden_states_32 + hidden_states_36 + hidden_states_32 = hidden_states_36 = None + hidden_states_38 = torch.nn.functional.layer_norm( + hidden_states_37, + (1024,), + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_37 = ( + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_ + ) = None + linear_30 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_18 = linear_30.view(1, 11, -1, 64) + linear_30 = None + query_states_6 = view_18.transpose(1, 2) + view_18 = None + key_states_12 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_12 = torch._C._nn.linear( + hidden_states_38, + l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_19 = key_states_12.view(1, 11, -1, 64) + key_states_12 = None + key_states_13 = view_19.transpose(1, 2) + view_19 = None + view_20 = value_states_12.view(1, 11, -1, 64) + value_states_12 = None + value_states_13 = view_20.transpose(1, 2) + view_20 = None + attention_mask_6 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_6 = query_states_6.contiguous() + query_states_6 = None + key_6 = key_states_13.contiguous() + value_6 = value_states_13.contiguous() + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_6, + key_6, + value_6, + attn_mask=attention_mask_6, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_6 = key_6 = value_6 = attention_mask_6 = None + transpose_27 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_25 = transpose_27.contiguous() + transpose_27 = None + reshape_7 = attn_output_25.reshape(1, 11, -1) + attn_output_25 = None + attn_output_26 = reshape_7.contiguous() + reshape_7 = None + attn_output_27 = torch._C._nn.linear( + attn_output_26, + l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_26 = l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_39 = torch.nn.functional.dropout( + attn_output_27, p=0.1, training=False + ) + attn_output_27 = None + hidden_states_40 = hidden_states_38 + hidden_states_39 + hidden_states_38 = hidden_states_39 = None + hidden_states_41 = torch.nn.functional.layer_norm( + hidden_states_40, + (1024,), + l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_40 = l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_34 = torch._C._nn.linear( + hidden_states_41, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_21 = linear_34.view(1, 11, -1, 64) + linear_34 = None + query_states_7 = view_21.transpose(1, 2) + view_21 = None + key_states_14 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_14 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_22 = key_states_14.view(1, 11, -1, 64) + key_states_14 = None + key_states_15 = view_22.transpose(1, 2) + view_22 = None + view_23 = value_states_14.view(1, 11, -1, 64) + value_states_14 = None + value_states_15 = view_23.transpose(1, 2) + view_23 = None + attention_mask_7 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_7 = query_states_7.contiguous() + query_states_7 = None + key_7 = key_states_15.contiguous() + value_7 = value_states_15.contiguous() + attn_output_28 = torch._C._nn.scaled_dot_product_attention( + query_7, + key_7, + value_7, + attn_mask=attention_mask_7, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_7 = key_7 = value_7 = attention_mask_7 = None + transpose_31 = attn_output_28.transpose(1, 2) + attn_output_28 = None + attn_output_29 = transpose_31.contiguous() + transpose_31 = None + reshape_8 = attn_output_29.reshape(1, 11, -1) + attn_output_29 = None + attn_output_30 = reshape_8.contiguous() + reshape_8 = None + attn_output_31 = torch._C._nn.linear( + attn_output_30, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_30 = l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_42 = torch.nn.functional.dropout( + attn_output_31, p=0.1, training=False + ) + attn_output_31 = None + hidden_states_43 = hidden_states_41 + hidden_states_42 + hidden_states_41 = hidden_states_42 = None + hidden_states_44 = torch.nn.functional.layer_norm( + hidden_states_43, + (1024,), + l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_43 = l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_38 = torch._C._nn.linear( + hidden_states_44, + l_self_modules_layers_modules_3_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_3_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_3_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_3_modules_fc1_parameters_bias_ + ) = None + hidden_states_45 = torch._C._nn.gelu(linear_38) + linear_38 = None + hidden_states_46 = torch.nn.functional.dropout( + hidden_states_45, p=0.0, training=False + ) + hidden_states_45 = None + hidden_states_47 = torch._C._nn.linear( + hidden_states_46, + l_self_modules_layers_modules_3_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_3_modules_fc2_parameters_bias_, + ) + hidden_states_46 = ( + l_self_modules_layers_modules_3_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_3_modules_fc2_parameters_bias_ = None + hidden_states_48 = torch.nn.functional.dropout( + hidden_states_47, p=0.1, training=False + ) + hidden_states_47 = None + hidden_states_49 = hidden_states_44 + hidden_states_48 + hidden_states_44 = hidden_states_48 = None + hidden_states_50 = torch.nn.functional.layer_norm( + hidden_states_49, + (1024,), + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_49 = ( + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_ + ) = None + linear_40 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_24 = linear_40.view(1, 11, -1, 64) + linear_40 = None + query_states_8 = view_24.transpose(1, 2) + view_24 = None + key_states_16 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_16 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_25 = key_states_16.view(1, 11, -1, 64) + key_states_16 = None + key_states_17 = view_25.transpose(1, 2) + view_25 = None + view_26 = value_states_16.view(1, 11, -1, 64) + value_states_16 = None + value_states_17 = view_26.transpose(1, 2) + view_26 = None + attention_mask_8 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_8 = query_states_8.contiguous() + query_states_8 = None + key_8 = key_states_17.contiguous() + value_8 = value_states_17.contiguous() + attn_output_32 = torch._C._nn.scaled_dot_product_attention( + query_8, + key_8, + value_8, + attn_mask=attention_mask_8, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_8 = key_8 = value_8 = attention_mask_8 = None + transpose_35 = attn_output_32.transpose(1, 2) + attn_output_32 = None + attn_output_33 = transpose_35.contiguous() + transpose_35 = None + reshape_9 = attn_output_33.reshape(1, 11, -1) + attn_output_33 = None + attn_output_34 = reshape_9.contiguous() + reshape_9 = None + attn_output_35 = torch._C._nn.linear( + attn_output_34, + l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_34 = l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_51 = torch.nn.functional.dropout( + attn_output_35, p=0.1, training=False + ) + attn_output_35 = None + hidden_states_52 = hidden_states_50 + hidden_states_51 + hidden_states_50 = hidden_states_51 = None + hidden_states_53 = torch.nn.functional.layer_norm( + hidden_states_52, + (1024,), + l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_52 = l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_44 = torch._C._nn.linear( + hidden_states_53, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_27 = linear_44.view(1, 11, -1, 64) + linear_44 = None + query_states_9 = view_27.transpose(1, 2) + view_27 = None + key_states_18 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_18 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_28 = key_states_18.view(1, 11, -1, 64) + key_states_18 = None + key_states_19 = view_28.transpose(1, 2) + view_28 = None + view_29 = value_states_18.view(1, 11, -1, 64) + value_states_18 = None + value_states_19 = view_29.transpose(1, 2) + view_29 = None + attention_mask_9 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_9 = query_states_9.contiguous() + query_states_9 = None + key_9 = key_states_19.contiguous() + value_9 = value_states_19.contiguous() + attn_output_36 = torch._C._nn.scaled_dot_product_attention( + query_9, + key_9, + value_9, + attn_mask=attention_mask_9, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_9 = key_9 = value_9 = attention_mask_9 = None + transpose_39 = attn_output_36.transpose(1, 2) + attn_output_36 = None + attn_output_37 = transpose_39.contiguous() + transpose_39 = None + reshape_10 = attn_output_37.reshape(1, 11, -1) + attn_output_37 = None + attn_output_38 = reshape_10.contiguous() + reshape_10 = None + attn_output_39 = torch._C._nn.linear( + attn_output_38, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_38 = l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_54 = torch.nn.functional.dropout( + attn_output_39, p=0.1, training=False + ) + attn_output_39 = None + hidden_states_55 = hidden_states_53 + hidden_states_54 + hidden_states_53 = hidden_states_54 = None + hidden_states_56 = torch.nn.functional.layer_norm( + hidden_states_55, + (1024,), + l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_55 = l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_48 = torch._C._nn.linear( + hidden_states_56, + l_self_modules_layers_modules_4_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_4_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_4_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_4_modules_fc1_parameters_bias_ + ) = None + hidden_states_57 = torch._C._nn.gelu(linear_48) + linear_48 = None + hidden_states_58 = torch.nn.functional.dropout( + hidden_states_57, p=0.0, training=False + ) + hidden_states_57 = None + hidden_states_59 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_layers_modules_4_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_4_modules_fc2_parameters_bias_, + ) + hidden_states_58 = ( + l_self_modules_layers_modules_4_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_4_modules_fc2_parameters_bias_ = None + hidden_states_60 = torch.nn.functional.dropout( + hidden_states_59, p=0.1, training=False + ) + hidden_states_59 = None + hidden_states_61 = hidden_states_56 + hidden_states_60 + hidden_states_56 = hidden_states_60 = None + hidden_states_62 = torch.nn.functional.layer_norm( + hidden_states_61, + (1024,), + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_61 = ( + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_ + ) = None + linear_50 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_30 = linear_50.view(1, 11, -1, 64) + linear_50 = None + query_states_10 = view_30.transpose(1, 2) + view_30 = None + key_states_20 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_20 = torch._C._nn.linear( + hidden_states_62, + l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_31 = key_states_20.view(1, 11, -1, 64) + key_states_20 = None + key_states_21 = view_31.transpose(1, 2) + view_31 = None + view_32 = value_states_20.view(1, 11, -1, 64) + value_states_20 = None + value_states_21 = view_32.transpose(1, 2) + view_32 = None + attention_mask_10 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_10 = query_states_10.contiguous() + query_states_10 = None + key_10 = key_states_21.contiguous() + value_10 = value_states_21.contiguous() + attn_output_40 = torch._C._nn.scaled_dot_product_attention( + query_10, + key_10, + value_10, + attn_mask=attention_mask_10, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_10 = key_10 = value_10 = attention_mask_10 = None + transpose_43 = attn_output_40.transpose(1, 2) + attn_output_40 = None + attn_output_41 = transpose_43.contiguous() + transpose_43 = None + reshape_11 = attn_output_41.reshape(1, 11, -1) + attn_output_41 = None + attn_output_42 = reshape_11.contiguous() + reshape_11 = None + attn_output_43 = torch._C._nn.linear( + attn_output_42, + l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_42 = l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_63 = torch.nn.functional.dropout( + attn_output_43, p=0.1, training=False + ) + attn_output_43 = None + hidden_states_64 = hidden_states_62 + hidden_states_63 + hidden_states_62 = hidden_states_63 = None + hidden_states_65 = torch.nn.functional.layer_norm( + hidden_states_64, + (1024,), + l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_64 = l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_54 = torch._C._nn.linear( + hidden_states_65, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_33 = linear_54.view(1, 11, -1, 64) + linear_54 = None + query_states_11 = view_33.transpose(1, 2) + view_33 = None + key_states_22 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_22 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_34 = key_states_22.view(1, 11, -1, 64) + key_states_22 = None + key_states_23 = view_34.transpose(1, 2) + view_34 = None + view_35 = value_states_22.view(1, 11, -1, 64) + value_states_22 = None + value_states_23 = view_35.transpose(1, 2) + view_35 = None + attention_mask_11 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_11 = query_states_11.contiguous() + query_states_11 = None + key_11 = key_states_23.contiguous() + value_11 = value_states_23.contiguous() + attn_output_44 = torch._C._nn.scaled_dot_product_attention( + query_11, + key_11, + value_11, + attn_mask=attention_mask_11, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_11 = key_11 = value_11 = attention_mask_11 = None + transpose_47 = attn_output_44.transpose(1, 2) + attn_output_44 = None + attn_output_45 = transpose_47.contiguous() + transpose_47 = None + reshape_12 = attn_output_45.reshape(1, 11, -1) + attn_output_45 = None + attn_output_46 = reshape_12.contiguous() + reshape_12 = None + attn_output_47 = torch._C._nn.linear( + attn_output_46, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_46 = l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_66 = torch.nn.functional.dropout( + attn_output_47, p=0.1, training=False + ) + attn_output_47 = None + hidden_states_67 = hidden_states_65 + hidden_states_66 + hidden_states_65 = hidden_states_66 = None + hidden_states_68 = torch.nn.functional.layer_norm( + hidden_states_67, + (1024,), + l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_67 = l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_58 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_layers_modules_5_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_5_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_5_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_5_modules_fc1_parameters_bias_ + ) = None + hidden_states_69 = torch._C._nn.gelu(linear_58) + linear_58 = None + hidden_states_70 = torch.nn.functional.dropout( + hidden_states_69, p=0.0, training=False + ) + hidden_states_69 = None + hidden_states_71 = torch._C._nn.linear( + hidden_states_70, + l_self_modules_layers_modules_5_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_5_modules_fc2_parameters_bias_, + ) + hidden_states_70 = ( + l_self_modules_layers_modules_5_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_5_modules_fc2_parameters_bias_ = None + hidden_states_72 = torch.nn.functional.dropout( + hidden_states_71, p=0.1, training=False + ) + hidden_states_71 = None + hidden_states_73 = hidden_states_68 + hidden_states_72 + hidden_states_68 = hidden_states_72 = None + hidden_states_74 = torch.nn.functional.layer_norm( + hidden_states_73, + (1024,), + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_73 = ( + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_ + ) = None + linear_60 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_36 = linear_60.view(1, 11, -1, 64) + linear_60 = None + query_states_12 = view_36.transpose(1, 2) + view_36 = None + key_states_24 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_24 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_37 = key_states_24.view(1, 11, -1, 64) + key_states_24 = None + key_states_25 = view_37.transpose(1, 2) + view_37 = None + view_38 = value_states_24.view(1, 11, -1, 64) + value_states_24 = None + value_states_25 = view_38.transpose(1, 2) + view_38 = None + attention_mask_12 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_12 = query_states_12.contiguous() + query_states_12 = None + key_12 = key_states_25.contiguous() + value_12 = value_states_25.contiguous() + attn_output_48 = torch._C._nn.scaled_dot_product_attention( + query_12, + key_12, + value_12, + attn_mask=attention_mask_12, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_12 = key_12 = value_12 = attention_mask_12 = None + transpose_51 = attn_output_48.transpose(1, 2) + attn_output_48 = None + attn_output_49 = transpose_51.contiguous() + transpose_51 = None + reshape_13 = attn_output_49.reshape(1, 11, -1) + attn_output_49 = None + attn_output_50 = reshape_13.contiguous() + reshape_13 = None + attn_output_51 = torch._C._nn.linear( + attn_output_50, + l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_50 = l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_75 = torch.nn.functional.dropout( + attn_output_51, p=0.1, training=False + ) + attn_output_51 = None + hidden_states_76 = hidden_states_74 + hidden_states_75 + hidden_states_74 = hidden_states_75 = None + hidden_states_77 = torch.nn.functional.layer_norm( + hidden_states_76, + (1024,), + l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_76 = l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_64 = torch._C._nn.linear( + hidden_states_77, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_39 = linear_64.view(1, 11, -1, 64) + linear_64 = None + query_states_13 = view_39.transpose(1, 2) + view_39 = None + key_states_26 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_26 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_40 = key_states_26.view(1, 11, -1, 64) + key_states_26 = None + key_states_27 = view_40.transpose(1, 2) + view_40 = None + view_41 = value_states_26.view(1, 11, -1, 64) + value_states_26 = None + value_states_27 = view_41.transpose(1, 2) + view_41 = None + attention_mask_13 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_13 = query_states_13.contiguous() + query_states_13 = None + key_13 = key_states_27.contiguous() + value_13 = value_states_27.contiguous() + attn_output_52 = torch._C._nn.scaled_dot_product_attention( + query_13, + key_13, + value_13, + attn_mask=attention_mask_13, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_13 = key_13 = value_13 = attention_mask_13 = None + transpose_55 = attn_output_52.transpose(1, 2) + attn_output_52 = None + attn_output_53 = transpose_55.contiguous() + transpose_55 = None + reshape_14 = attn_output_53.reshape(1, 11, -1) + attn_output_53 = None + attn_output_54 = reshape_14.contiguous() + reshape_14 = None + attn_output_55 = torch._C._nn.linear( + attn_output_54, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_54 = l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_78 = torch.nn.functional.dropout( + attn_output_55, p=0.1, training=False + ) + attn_output_55 = None + hidden_states_79 = hidden_states_77 + hidden_states_78 + hidden_states_77 = hidden_states_78 = None + hidden_states_80 = torch.nn.functional.layer_norm( + hidden_states_79, + (1024,), + l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_79 = l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_68 = torch._C._nn.linear( + hidden_states_80, + l_self_modules_layers_modules_6_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_6_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_6_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_6_modules_fc1_parameters_bias_ + ) = None + hidden_states_81 = torch._C._nn.gelu(linear_68) + linear_68 = None + hidden_states_82 = torch.nn.functional.dropout( + hidden_states_81, p=0.0, training=False + ) + hidden_states_81 = None + hidden_states_83 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_layers_modules_6_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_6_modules_fc2_parameters_bias_, + ) + hidden_states_82 = ( + l_self_modules_layers_modules_6_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_6_modules_fc2_parameters_bias_ = None + hidden_states_84 = torch.nn.functional.dropout( + hidden_states_83, p=0.1, training=False + ) + hidden_states_83 = None + hidden_states_85 = hidden_states_80 + hidden_states_84 + hidden_states_80 = hidden_states_84 = None + hidden_states_86 = torch.nn.functional.layer_norm( + hidden_states_85, + (1024,), + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_85 = ( + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_ + ) = None + linear_70 = torch._C._nn.linear( + hidden_states_86, + l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_42 = linear_70.view(1, 11, -1, 64) + linear_70 = None + query_states_14 = view_42.transpose(1, 2) + view_42 = None + key_states_28 = torch._C._nn.linear( + hidden_states_86, + l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_28 = torch._C._nn.linear( + hidden_states_86, + l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_43 = key_states_28.view(1, 11, -1, 64) + key_states_28 = None + key_states_29 = view_43.transpose(1, 2) + view_43 = None + view_44 = value_states_28.view(1, 11, -1, 64) + value_states_28 = None + value_states_29 = view_44.transpose(1, 2) + view_44 = None + attention_mask_14 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_14 = query_states_14.contiguous() + query_states_14 = None + key_14 = key_states_29.contiguous() + value_14 = value_states_29.contiguous() + attn_output_56 = torch._C._nn.scaled_dot_product_attention( + query_14, + key_14, + value_14, + attn_mask=attention_mask_14, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_14 = key_14 = value_14 = attention_mask_14 = None + transpose_59 = attn_output_56.transpose(1, 2) + attn_output_56 = None + attn_output_57 = transpose_59.contiguous() + transpose_59 = None + reshape_15 = attn_output_57.reshape(1, 11, -1) + attn_output_57 = None + attn_output_58 = reshape_15.contiguous() + reshape_15 = None + attn_output_59 = torch._C._nn.linear( + attn_output_58, + l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_58 = l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_87 = torch.nn.functional.dropout( + attn_output_59, p=0.1, training=False + ) + attn_output_59 = None + hidden_states_88 = hidden_states_86 + hidden_states_87 + hidden_states_86 = hidden_states_87 = None + hidden_states_89 = torch.nn.functional.layer_norm( + hidden_states_88, + (1024,), + l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_88 = l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_74 = torch._C._nn.linear( + hidden_states_89, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_45 = linear_74.view(1, 11, -1, 64) + linear_74 = None + query_states_15 = view_45.transpose(1, 2) + view_45 = None + key_states_30 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_30 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_46 = key_states_30.view(1, 11, -1, 64) + key_states_30 = None + key_states_31 = view_46.transpose(1, 2) + view_46 = None + view_47 = value_states_30.view(1, 11, -1, 64) + value_states_30 = None + value_states_31 = view_47.transpose(1, 2) + view_47 = None + attention_mask_15 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_15 = query_states_15.contiguous() + query_states_15 = None + key_15 = key_states_31.contiguous() + value_15 = value_states_31.contiguous() + attn_output_60 = torch._C._nn.scaled_dot_product_attention( + query_15, + key_15, + value_15, + attn_mask=attention_mask_15, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_15 = key_15 = value_15 = attention_mask_15 = None + transpose_63 = attn_output_60.transpose(1, 2) + attn_output_60 = None + attn_output_61 = transpose_63.contiguous() + transpose_63 = None + reshape_16 = attn_output_61.reshape(1, 11, -1) + attn_output_61 = None + attn_output_62 = reshape_16.contiguous() + reshape_16 = None + attn_output_63 = torch._C._nn.linear( + attn_output_62, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_62 = l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_90 = torch.nn.functional.dropout( + attn_output_63, p=0.1, training=False + ) + attn_output_63 = None + hidden_states_91 = hidden_states_89 + hidden_states_90 + hidden_states_89 = hidden_states_90 = None + hidden_states_92 = torch.nn.functional.layer_norm( + hidden_states_91, + (1024,), + l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_91 = l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_78 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_layers_modules_7_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_7_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_7_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_7_modules_fc1_parameters_bias_ + ) = None + hidden_states_93 = torch._C._nn.gelu(linear_78) + linear_78 = None + hidden_states_94 = torch.nn.functional.dropout( + hidden_states_93, p=0.0, training=False + ) + hidden_states_93 = None + hidden_states_95 = torch._C._nn.linear( + hidden_states_94, + l_self_modules_layers_modules_7_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_7_modules_fc2_parameters_bias_, + ) + hidden_states_94 = ( + l_self_modules_layers_modules_7_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_7_modules_fc2_parameters_bias_ = None + hidden_states_96 = torch.nn.functional.dropout( + hidden_states_95, p=0.1, training=False + ) + hidden_states_95 = None + hidden_states_97 = hidden_states_92 + hidden_states_96 + hidden_states_92 = hidden_states_96 = None + hidden_states_98 = torch.nn.functional.layer_norm( + hidden_states_97, + (1024,), + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_97 = ( + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_ + ) = None + linear_80 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_48 = linear_80.view(1, 11, -1, 64) + linear_80 = None + query_states_16 = view_48.transpose(1, 2) + view_48 = None + key_states_32 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_32 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_49 = key_states_32.view(1, 11, -1, 64) + key_states_32 = None + key_states_33 = view_49.transpose(1, 2) + view_49 = None + view_50 = value_states_32.view(1, 11, -1, 64) + value_states_32 = None + value_states_33 = view_50.transpose(1, 2) + view_50 = None + attention_mask_16 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_16 = query_states_16.contiguous() + query_states_16 = None + key_16 = key_states_33.contiguous() + value_16 = value_states_33.contiguous() + attn_output_64 = torch._C._nn.scaled_dot_product_attention( + query_16, + key_16, + value_16, + attn_mask=attention_mask_16, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_16 = key_16 = value_16 = attention_mask_16 = None + transpose_67 = attn_output_64.transpose(1, 2) + attn_output_64 = None + attn_output_65 = transpose_67.contiguous() + transpose_67 = None + reshape_17 = attn_output_65.reshape(1, 11, -1) + attn_output_65 = None + attn_output_66 = reshape_17.contiguous() + reshape_17 = None + attn_output_67 = torch._C._nn.linear( + attn_output_66, + l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_66 = l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_99 = torch.nn.functional.dropout( + attn_output_67, p=0.1, training=False + ) + attn_output_67 = None + hidden_states_100 = hidden_states_98 + hidden_states_99 + hidden_states_98 = hidden_states_99 = None + hidden_states_101 = torch.nn.functional.layer_norm( + hidden_states_100, + (1024,), + l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_100 = l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_84 = torch._C._nn.linear( + hidden_states_101, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_51 = linear_84.view(1, 11, -1, 64) + linear_84 = None + query_states_17 = view_51.transpose(1, 2) + view_51 = None + key_states_34 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_34 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_52 = key_states_34.view(1, 11, -1, 64) + key_states_34 = None + key_states_35 = view_52.transpose(1, 2) + view_52 = None + view_53 = value_states_34.view(1, 11, -1, 64) + value_states_34 = None + value_states_35 = view_53.transpose(1, 2) + view_53 = None + attention_mask_17 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_17 = query_states_17.contiguous() + query_states_17 = None + key_17 = key_states_35.contiguous() + value_17 = value_states_35.contiguous() + attn_output_68 = torch._C._nn.scaled_dot_product_attention( + query_17, + key_17, + value_17, + attn_mask=attention_mask_17, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_17 = key_17 = value_17 = attention_mask_17 = None + transpose_71 = attn_output_68.transpose(1, 2) + attn_output_68 = None + attn_output_69 = transpose_71.contiguous() + transpose_71 = None + reshape_18 = attn_output_69.reshape(1, 11, -1) + attn_output_69 = None + attn_output_70 = reshape_18.contiguous() + reshape_18 = None + attn_output_71 = torch._C._nn.linear( + attn_output_70, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_70 = l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_102 = torch.nn.functional.dropout( + attn_output_71, p=0.1, training=False + ) + attn_output_71 = None + hidden_states_103 = hidden_states_101 + hidden_states_102 + hidden_states_101 = hidden_states_102 = None + hidden_states_104 = torch.nn.functional.layer_norm( + hidden_states_103, + (1024,), + l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_103 = l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_88 = torch._C._nn.linear( + hidden_states_104, + l_self_modules_layers_modules_8_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_8_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_8_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_8_modules_fc1_parameters_bias_ + ) = None + hidden_states_105 = torch._C._nn.gelu(linear_88) + linear_88 = None + hidden_states_106 = torch.nn.functional.dropout( + hidden_states_105, p=0.0, training=False + ) + hidden_states_105 = None + hidden_states_107 = torch._C._nn.linear( + hidden_states_106, + l_self_modules_layers_modules_8_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_8_modules_fc2_parameters_bias_, + ) + hidden_states_106 = ( + l_self_modules_layers_modules_8_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_8_modules_fc2_parameters_bias_ = None + hidden_states_108 = torch.nn.functional.dropout( + hidden_states_107, p=0.1, training=False + ) + hidden_states_107 = None + hidden_states_109 = hidden_states_104 + hidden_states_108 + hidden_states_104 = hidden_states_108 = None + hidden_states_110 = torch.nn.functional.layer_norm( + hidden_states_109, + (1024,), + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_109 = ( + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_ + ) = None + linear_90 = torch._C._nn.linear( + hidden_states_110, + l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_54 = linear_90.view(1, 11, -1, 64) + linear_90 = None + query_states_18 = view_54.transpose(1, 2) + view_54 = None + key_states_36 = torch._C._nn.linear( + hidden_states_110, + l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_36 = torch._C._nn.linear( + hidden_states_110, + l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_55 = key_states_36.view(1, 11, -1, 64) + key_states_36 = None + key_states_37 = view_55.transpose(1, 2) + view_55 = None + view_56 = value_states_36.view(1, 11, -1, 64) + value_states_36 = None + value_states_37 = view_56.transpose(1, 2) + view_56 = None + attention_mask_18 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_18 = query_states_18.contiguous() + query_states_18 = None + key_18 = key_states_37.contiguous() + value_18 = value_states_37.contiguous() + attn_output_72 = torch._C._nn.scaled_dot_product_attention( + query_18, + key_18, + value_18, + attn_mask=attention_mask_18, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_18 = key_18 = value_18 = attention_mask_18 = None + transpose_75 = attn_output_72.transpose(1, 2) + attn_output_72 = None + attn_output_73 = transpose_75.contiguous() + transpose_75 = None + reshape_19 = attn_output_73.reshape(1, 11, -1) + attn_output_73 = None + attn_output_74 = reshape_19.contiguous() + reshape_19 = None + attn_output_75 = torch._C._nn.linear( + attn_output_74, + l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_74 = l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_111 = torch.nn.functional.dropout( + attn_output_75, p=0.1, training=False + ) + attn_output_75 = None + hidden_states_112 = hidden_states_110 + hidden_states_111 + hidden_states_110 = hidden_states_111 = None + hidden_states_113 = torch.nn.functional.layer_norm( + hidden_states_112, + (1024,), + l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_112 = l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_94 = torch._C._nn.linear( + hidden_states_113, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_57 = linear_94.view(1, 11, -1, 64) + linear_94 = None + query_states_19 = view_57.transpose(1, 2) + view_57 = None + key_states_38 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_38 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_58 = key_states_38.view(1, 11, -1, 64) + key_states_38 = None + key_states_39 = view_58.transpose(1, 2) + view_58 = None + view_59 = value_states_38.view(1, 11, -1, 64) + value_states_38 = None + value_states_39 = view_59.transpose(1, 2) + view_59 = None + attention_mask_19 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_19 = query_states_19.contiguous() + query_states_19 = None + key_19 = key_states_39.contiguous() + value_19 = value_states_39.contiguous() + attn_output_76 = torch._C._nn.scaled_dot_product_attention( + query_19, + key_19, + value_19, + attn_mask=attention_mask_19, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_19 = key_19 = value_19 = attention_mask_19 = None + transpose_79 = attn_output_76.transpose(1, 2) + attn_output_76 = None + attn_output_77 = transpose_79.contiguous() + transpose_79 = None + reshape_20 = attn_output_77.reshape(1, 11, -1) + attn_output_77 = None + attn_output_78 = reshape_20.contiguous() + reshape_20 = None + attn_output_79 = torch._C._nn.linear( + attn_output_78, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_78 = l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_114 = torch.nn.functional.dropout( + attn_output_79, p=0.1, training=False + ) + attn_output_79 = None + hidden_states_115 = hidden_states_113 + hidden_states_114 + hidden_states_113 = hidden_states_114 = None + hidden_states_116 = torch.nn.functional.layer_norm( + hidden_states_115, + (1024,), + l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_115 = l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_98 = torch._C._nn.linear( + hidden_states_116, + l_self_modules_layers_modules_9_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_9_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_9_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_9_modules_fc1_parameters_bias_ + ) = None + hidden_states_117 = torch._C._nn.gelu(linear_98) + linear_98 = None + hidden_states_118 = torch.nn.functional.dropout( + hidden_states_117, p=0.0, training=False + ) + hidden_states_117 = None + hidden_states_119 = torch._C._nn.linear( + hidden_states_118, + l_self_modules_layers_modules_9_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_9_modules_fc2_parameters_bias_, + ) + hidden_states_118 = ( + l_self_modules_layers_modules_9_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_9_modules_fc2_parameters_bias_ = None + hidden_states_120 = torch.nn.functional.dropout( + hidden_states_119, p=0.1, training=False + ) + hidden_states_119 = None + hidden_states_121 = hidden_states_116 + hidden_states_120 + hidden_states_116 = hidden_states_120 = None + hidden_states_122 = torch.nn.functional.layer_norm( + hidden_states_121, + (1024,), + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_121 = ( + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_ + ) = None + linear_100 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_60 = linear_100.view(1, 11, -1, 64) + linear_100 = None + query_states_20 = view_60.transpose(1, 2) + view_60 = None + key_states_40 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_40 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_61 = key_states_40.view(1, 11, -1, 64) + key_states_40 = None + key_states_41 = view_61.transpose(1, 2) + view_61 = None + view_62 = value_states_40.view(1, 11, -1, 64) + value_states_40 = None + value_states_41 = view_62.transpose(1, 2) + view_62 = None + attention_mask_20 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_20 = query_states_20.contiguous() + query_states_20 = None + key_20 = key_states_41.contiguous() + value_20 = value_states_41.contiguous() + attn_output_80 = torch._C._nn.scaled_dot_product_attention( + query_20, + key_20, + value_20, + attn_mask=attention_mask_20, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_20 = key_20 = value_20 = attention_mask_20 = None + transpose_83 = attn_output_80.transpose(1, 2) + attn_output_80 = None + attn_output_81 = transpose_83.contiguous() + transpose_83 = None + reshape_21 = attn_output_81.reshape(1, 11, -1) + attn_output_81 = None + attn_output_82 = reshape_21.contiguous() + reshape_21 = None + attn_output_83 = torch._C._nn.linear( + attn_output_82, + l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_82 = l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_123 = torch.nn.functional.dropout( + attn_output_83, p=0.1, training=False + ) + attn_output_83 = None + hidden_states_124 = hidden_states_122 + hidden_states_123 + hidden_states_122 = hidden_states_123 = None + hidden_states_125 = torch.nn.functional.layer_norm( + hidden_states_124, + (1024,), + l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_124 = l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_104 = torch._C._nn.linear( + hidden_states_125, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_63 = linear_104.view(1, 11, -1, 64) + linear_104 = None + query_states_21 = view_63.transpose(1, 2) + view_63 = None + key_states_42 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_42 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_64 = key_states_42.view(1, 11, -1, 64) + key_states_42 = None + key_states_43 = view_64.transpose(1, 2) + view_64 = None + view_65 = value_states_42.view(1, 11, -1, 64) + value_states_42 = None + value_states_43 = view_65.transpose(1, 2) + view_65 = None + attention_mask_21 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + query_21 = query_states_21.contiguous() + query_states_21 = None + key_21 = key_states_43.contiguous() + value_21 = value_states_43.contiguous() + attn_output_84 = torch._C._nn.scaled_dot_product_attention( + query_21, + key_21, + value_21, + attn_mask=attention_mask_21, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_21 = key_21 = value_21 = attention_mask_21 = None + transpose_87 = attn_output_84.transpose(1, 2) + attn_output_84 = None + attn_output_85 = transpose_87.contiguous() + transpose_87 = None + reshape_22 = attn_output_85.reshape(1, 11, -1) + attn_output_85 = None + attn_output_86 = reshape_22.contiguous() + reshape_22 = None + attn_output_87 = torch._C._nn.linear( + attn_output_86, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_86 = l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_126 = torch.nn.functional.dropout( + attn_output_87, p=0.1, training=False + ) + attn_output_87 = None + hidden_states_127 = hidden_states_125 + hidden_states_126 + hidden_states_125 = hidden_states_126 = None + hidden_states_128 = torch.nn.functional.layer_norm( + hidden_states_127, + (1024,), + l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_127 = l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_108 = torch._C._nn.linear( + hidden_states_128, + l_self_modules_layers_modules_10_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_10_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_10_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_10_modules_fc1_parameters_bias_ + ) = None + hidden_states_129 = torch._C._nn.gelu(linear_108) + linear_108 = None + hidden_states_130 = torch.nn.functional.dropout( + hidden_states_129, p=0.0, training=False + ) + hidden_states_129 = None + hidden_states_131 = torch._C._nn.linear( + hidden_states_130, + l_self_modules_layers_modules_10_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_10_modules_fc2_parameters_bias_, + ) + hidden_states_130 = ( + l_self_modules_layers_modules_10_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_10_modules_fc2_parameters_bias_ = None + hidden_states_132 = torch.nn.functional.dropout( + hidden_states_131, p=0.1, training=False + ) + hidden_states_131 = None + hidden_states_133 = hidden_states_128 + hidden_states_132 + hidden_states_128 = hidden_states_132 = None + hidden_states_134 = torch.nn.functional.layer_norm( + hidden_states_133, + (1024,), + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_133 = ( + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_ + ) = None + linear_110 = torch._C._nn.linear( + hidden_states_134, + l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_ = (None) + view_66 = linear_110.view(1, 11, -1, 64) + linear_110 = None + query_states_22 = view_66.transpose(1, 2) + view_66 = None + key_states_44 = torch._C._nn.linear( + hidden_states_134, + l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_ = (None) + value_states_44 = torch._C._nn.linear( + hidden_states_134, + l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_ = (None) + view_67 = key_states_44.view(1, 11, -1, 64) + key_states_44 = None + key_states_45 = view_67.transpose(1, 2) + view_67 = None + view_68 = value_states_44.view(1, 11, -1, 64) + value_states_44 = None + value_states_45 = view_68.transpose(1, 2) + view_68 = None + attention_mask_22 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + causal_mask_3 = None + query_22 = query_states_22.contiguous() + query_states_22 = None + key_22 = key_states_45.contiguous() + value_22 = value_states_45.contiguous() + attn_output_88 = torch._C._nn.scaled_dot_product_attention( + query_22, + key_22, + value_22, + attn_mask=attention_mask_22, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_22 = key_22 = value_22 = attention_mask_22 = None + transpose_91 = attn_output_88.transpose(1, 2) + attn_output_88 = None + attn_output_89 = transpose_91.contiguous() + transpose_91 = None + reshape_23 = attn_output_89.reshape(1, 11, -1) + attn_output_89 = None + attn_output_90 = reshape_23.contiguous() + reshape_23 = None + attn_output_91 = torch._C._nn.linear( + attn_output_90, + l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_, + ) + attn_output_90 = l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_135 = torch.nn.functional.dropout( + attn_output_91, p=0.1, training=False + ) + attn_output_91 = None + hidden_states_136 = hidden_states_134 + hidden_states_135 + hidden_states_134 = hidden_states_135 = None + hidden_states_137 = torch.nn.functional.layer_norm( + hidden_states_136, + (1024,), + l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_136 = l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_ = (None) + linear_114 = torch._C._nn.linear( + hidden_states_137, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_ = (None) + view_69 = linear_114.view(1, 11, -1, 64) + linear_114 = None + query_states_23 = view_69.transpose(1, 2) + view_69 = None + key_states_46 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_ = (None) + value_states_46 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_, + ) + l_encoder_hidden_states_ = l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_ = (None) + view_70 = key_states_46.view(1, 11, -1, 64) + key_states_46 = None + key_states_47 = view_70.transpose(1, 2) + view_70 = None + view_71 = value_states_46.view(1, 11, -1, 64) + value_states_46 = None + value_states_47 = view_71.transpose(1, 2) + view_71 = None + attention_mask_23 = encoder_attention_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 11, None), + ) + ] + encoder_attention_mask = None + query_23 = query_states_23.contiguous() + query_states_23 = None + key_23 = key_states_47.contiguous() + value_23 = value_states_47.contiguous() + attn_output_92 = torch._C._nn.scaled_dot_product_attention( + query_23, + key_23, + value_23, + attn_mask=attention_mask_23, + dropout_p=0.0, + scale=0.125, + is_causal=False, + ) + query_23 = key_23 = value_23 = attention_mask_23 = None + transpose_95 = attn_output_92.transpose(1, 2) + attn_output_92 = None + attn_output_93 = transpose_95.contiguous() + transpose_95 = None + reshape_24 = attn_output_93.reshape(1, 11, -1) + attn_output_93 = None + attn_output_94 = reshape_24.contiguous() + reshape_24 = None + attn_output_95 = torch._C._nn.linear( + attn_output_94, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_, + l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_, + ) + attn_output_94 = l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_ = l_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_ = (None) + hidden_states_138 = torch.nn.functional.dropout( + attn_output_95, p=0.1, training=False + ) + attn_output_95 = None + hidden_states_139 = hidden_states_137 + hidden_states_138 + hidden_states_137 = hidden_states_138 = None + hidden_states_140 = torch.nn.functional.layer_norm( + hidden_states_139, + (1024,), + l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_, + l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_139 = l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_ = l_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_ = (None) + linear_118 = torch._C._nn.linear( + hidden_states_140, + l_self_modules_layers_modules_11_modules_fc1_parameters_weight_, + l_self_modules_layers_modules_11_modules_fc1_parameters_bias_, + ) + l_self_modules_layers_modules_11_modules_fc1_parameters_weight_ = ( + l_self_modules_layers_modules_11_modules_fc1_parameters_bias_ + ) = None + hidden_states_141 = torch._C._nn.gelu(linear_118) + linear_118 = None + hidden_states_142 = torch.nn.functional.dropout( + hidden_states_141, p=0.0, training=False + ) + hidden_states_141 = None + hidden_states_143 = torch._C._nn.linear( + hidden_states_142, + l_self_modules_layers_modules_11_modules_fc2_parameters_weight_, + l_self_modules_layers_modules_11_modules_fc2_parameters_bias_, + ) + hidden_states_142 = ( + l_self_modules_layers_modules_11_modules_fc2_parameters_weight_ + ) = l_self_modules_layers_modules_11_modules_fc2_parameters_bias_ = None + hidden_states_144 = torch.nn.functional.dropout( + hidden_states_143, p=0.1, training=False + ) + hidden_states_143 = None + hidden_states_145 = hidden_states_140 + hidden_states_144 + hidden_states_140 = hidden_states_144 = None + hidden_states_146 = torch.nn.functional.layer_norm( + hidden_states_145, + (1024,), + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_, + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_, + 1e-05, + ) + hidden_states_145 = ( + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_ + ) = ( + l_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_ + ) = None + return ( + value_states_1, + key_states_1, + value_states_3, + key_states_3, + value_states_5, + key_states_5, + value_states_7, + key_states_7, + value_states_9, + key_states_9, + value_states_11, + key_states_11, + value_states_13, + key_states_13, + value_states_15, + key_states_15, + value_states_17, + key_states_17, + value_states_19, + key_states_19, + value_states_21, + key_states_21, + value_states_23, + key_states_23, + value_states_25, + key_states_25, + value_states_27, + key_states_27, + value_states_29, + key_states_29, + value_states_31, + key_states_31, + value_states_33, + key_states_33, + value_states_35, + key_states_35, + value_states_37, + key_states_37, + value_states_39, + key_states_39, + value_states_41, + key_states_41, + value_states_43, + key_states_43, + value_states_45, + key_states_45, + value_states_47, + key_states_47, + hidden_states_146, + ) diff --git a/samples/transformers-auto-model/facebook_bart-large-cnn/weight_meta.py b/samples/transformers-auto-model/facebook_bart-large-cnn/weight_meta.py new file mode 100644 index 0000000000..5cd6df3dd3 --- /dev/null +++ b/samples/transformers-auto-model/facebook_bart-large-cnn/weight_meta.py @@ -0,0 +1,3206 @@ +class Program_weight_tensor_meta_L_inputs_embeds_: + name = "L_inputs_embeds_" + shape = [1, 11, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_encoder_hidden_states_: + name = "L_encoder_hidden_states_" + shape = [1, 11, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_encoder_attention_mask_: + name = "L_encoder_attention_mask_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_embed_positions_parameters_weight_: + name = "L_self_modules_embed_positions_parameters_weight_" + shape = [1026, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layernorm_embedding_parameters_weight_: + name = "L_self_modules_layernorm_embedding_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.436 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layernorm_embedding_parameters_bias_: + name = "L_self_modules_layernorm_embedding_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.020 + std = 0.591 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.088 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.736 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_0_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.099 + std = 0.120 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.151 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.451 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.023 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.115 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.385 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_0_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.014 + std = 0.263 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.804 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_1_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.016 + std = 0.110 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.272 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.469 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.035 + std = 0.101 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.110 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.362 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_1_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.279 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.690 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_2_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.199 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.483 + std = 0.083 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.043 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.107 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.370 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_2_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.218 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.755 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_3_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.197 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.472 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.046 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.380 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_3_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.219 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.746 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_4_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.177 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.483 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.043 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.084 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.084 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.398 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_4_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.213 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.010 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.797 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_5_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.095 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.125 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.477 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.041 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.086 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.396 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_5_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.011 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.182 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.007 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.776 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_6_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.026 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.098 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.167 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.096 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.491 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.040 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.090 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.410 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_6_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.010 + std = 0.019 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.194 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.008 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.799 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_7_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.029 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.095 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.210 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.485 + std = 0.121 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.039 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.405 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_7_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.227 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.801 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_8_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.175 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.176 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.468 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.046 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.088 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.407 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_8_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.015 + std = 0.014 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.217 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.137 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.010 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.779 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_9_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.160 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.356 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.473 + std = 0.174 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.086 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.406 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_9_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.015 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.259 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.232 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.008 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.753 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_10_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.043 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.154 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.331 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.501 + std = 0.171 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.046 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.079 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_10_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_: + name = ( + "L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_weight_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.405 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_10_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.014 + std = 0.019 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.083 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.339 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 1.815 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_self_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.629 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layers_modules_11_modules_self_attn_layer_norm_parameters_bias_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_q_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.189 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_k_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.106 + std = 1.381 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_v_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_modules_out_proj_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.529 + std = 0.083 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_encoder_attn_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.062 + std = 0.125 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_fc1_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_fc1_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_fc1_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_fc1_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.050 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_fc2_parameters_weight_: + name = "L_self_modules_layers_modules_11_modules_fc2_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_fc2_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_fc2_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_: + name = ( + "L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_weight_" + ) + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.505 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_: + name = "L_self_modules_layers_modules_11_modules_final_layer_norm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.036 + std = 0.068 + data = None diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_hash.txt b/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_hash.txt new file mode 100644 index 0000000000..3e97045077 --- /dev/null +++ b/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_hash.txt @@ -0,0 +1 @@ +27301df7eb36e24c282ddad722d612ad349b2435a41cad08ed0c4ae214f93882 \ No newline at end of file diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_net.json b/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/google-bert_bert-large-uncased/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/input_meta.py b/samples/transformers-auto-model/google-bert_bert-large-uncased/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/input_tensor_constraints.py b/samples/transformers-auto-model/google-bert_bert-large-uncased/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/model.py b/samples/transformers-auto-model/google-bert_bert-large-uncased/model.py new file mode 100644 index 0000000000..6e27431e56 --- /dev/null +++ b/samples/transformers-auto-model/google-bert_bert-large-uncased/model.py @@ -0,0 +1,3080 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_token_type_ids_: torch.Tensor, + L_self_modules_embeddings_buffers_position_ids_: torch.Tensor, + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_attention_mask_: torch.Tensor, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_token_type_ids_ = L_token_type_ids_ + l_self_modules_embeddings_buffers_position_ids_ = ( + L_self_modules_embeddings_buffers_position_ids_ + ) + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_ + ) + l_attention_mask_ = L_attention_mask_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_pooler_modules_dense_parameters_weight_ = ( + L_self_modules_pooler_modules_dense_parameters_weight_ + ) + l_self_modules_pooler_modules_dense_parameters_bias_ = ( + L_self_modules_pooler_modules_dense_parameters_bias_ + ) + position_ids = l_self_modules_embeddings_buffers_position_ids_[ + (slice(None, None, None), slice(0, 11, None)) + ] + l_self_modules_embeddings_buffers_position_ids_ = None + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_, + 0, + None, + 2.0, + False, + False, + ) + l_input_ids_ = ( + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) = None + token_type_embeddings = torch.nn.functional.embedding( + l_token_type_ids_, + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_token_type_ids_ = ( + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) = None + embeddings = inputs_embeds + token_type_embeddings + inputs_embeds = token_type_embeddings = None + position_embeddings = torch.nn.functional.embedding( + position_ids, + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = ( + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) = None + embeddings += position_embeddings + embeddings_1 = embeddings + embeddings = position_embeddings = None + embeddings_2 = torch.nn.functional.layer_norm( + embeddings_1, + (1024,), + l_self_modules_embeddings_modules_layer_norm_parameters_weight_, + l_self_modules_embeddings_modules_layer_norm_parameters_bias_, + 1e-12, + ) + embeddings_1 = ( + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ + ) = l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = None + embeddings_3 = torch.nn.functional.dropout(embeddings_2, 0.1, False, False) + embeddings_2 = None + getitem_1 = l_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_attention_mask_ = None + expand = getitem_1.expand(1, 1, 11, 11) + getitem_1 = None + expanded_mask = expand.to(torch.float32) + expand = None + tensor = torch.tensor(1.0, dtype=torch.float32) + inverted_mask = tensor - expanded_mask + tensor = expanded_mask = None + to_1 = inverted_mask.to(torch.bool) + extended_attention_mask = inverted_mask.masked_fill( + to_1, -3.4028234663852886e38 + ) + inverted_mask = to_1 = None + linear = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view = linear.view(1, -1, 16, 64) + linear = None + query_layer = view.transpose(1, 2) + view = None + linear_1 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_1 = linear_1.view(1, -1, 16, 64) + linear_1 = None + key_layer = view_1.transpose(1, 2) + view_1 = None + linear_2 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_2 = linear_2.view(1, -1, 16, 64) + linear_2 = None + value_layer = view_2.transpose(1, 2) + view_2 = None + attn_output = torch._C._nn.scaled_dot_product_attention( + query_layer, + key_layer, + value_layer, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer = key_layer = value_layer = None + attn_output_1 = attn_output.transpose(1, 2) + attn_output = None + attn_output_2 = attn_output_1.reshape(1, 11, 1024) + attn_output_1 = None + hidden_states = torch._C._nn.linear( + attn_output_2, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_2 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_1 = torch.nn.functional.dropout(hidden_states, 0.1, False, False) + hidden_states = None + add_1 = hidden_states_1 + embeddings_3 + hidden_states_1 = embeddings_3 = None + hidden_states_2 = torch.nn.functional.layer_norm( + add_1, + (1024,), + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_1 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_3 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_4 = torch._C._nn.gelu(hidden_states_3) + hidden_states_3 = None + hidden_states_5 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_4 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_6 = torch.nn.functional.dropout( + hidden_states_5, 0.1, False, False + ) + hidden_states_5 = None + add_2 = hidden_states_6 + hidden_states_2 + hidden_states_6 = hidden_states_2 = None + hidden_states_7 = torch.nn.functional.layer_norm( + add_2, + (1024,), + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_2 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_6 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_3 = linear_6.view(1, -1, 16, 64) + linear_6 = None + query_layer_1 = view_3.transpose(1, 2) + view_3 = None + linear_7 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_4 = linear_7.view(1, -1, 16, 64) + linear_7 = None + key_layer_1 = view_4.transpose(1, 2) + view_4 = None + linear_8 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_5 = linear_8.view(1, -1, 16, 64) + linear_8 = None + value_layer_1 = view_5.transpose(1, 2) + view_5 = None + attn_output_3 = torch._C._nn.scaled_dot_product_attention( + query_layer_1, + key_layer_1, + value_layer_1, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_1 = key_layer_1 = value_layer_1 = None + attn_output_4 = attn_output_3.transpose(1, 2) + attn_output_3 = None + attn_output_5 = attn_output_4.reshape(1, 11, 1024) + attn_output_4 = None + hidden_states_8 = torch._C._nn.linear( + attn_output_5, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_9 = torch.nn.functional.dropout( + hidden_states_8, 0.1, False, False + ) + hidden_states_8 = None + add_3 = hidden_states_9 + hidden_states_7 + hidden_states_9 = hidden_states_7 = None + hidden_states_10 = torch.nn.functional.layer_norm( + add_3, + (1024,), + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_3 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_11 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_12 = torch._C._nn.gelu(hidden_states_11) + hidden_states_11 = None + hidden_states_13 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_12 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_14 = torch.nn.functional.dropout( + hidden_states_13, 0.1, False, False + ) + hidden_states_13 = None + add_4 = hidden_states_14 + hidden_states_10 + hidden_states_14 = hidden_states_10 = None + hidden_states_15 = torch.nn.functional.layer_norm( + add_4, + (1024,), + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_4 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_12 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_6 = linear_12.view(1, -1, 16, 64) + linear_12 = None + query_layer_2 = view_6.transpose(1, 2) + view_6 = None + linear_13 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_7 = linear_13.view(1, -1, 16, 64) + linear_13 = None + key_layer_2 = view_7.transpose(1, 2) + view_7 = None + linear_14 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_8 = linear_14.view(1, -1, 16, 64) + linear_14 = None + value_layer_2 = view_8.transpose(1, 2) + view_8 = None + attn_output_6 = torch._C._nn.scaled_dot_product_attention( + query_layer_2, + key_layer_2, + value_layer_2, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_2 = key_layer_2 = value_layer_2 = None + attn_output_7 = attn_output_6.transpose(1, 2) + attn_output_6 = None + attn_output_8 = attn_output_7.reshape(1, 11, 1024) + attn_output_7 = None + hidden_states_16 = torch._C._nn.linear( + attn_output_8, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_8 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_17 = torch.nn.functional.dropout( + hidden_states_16, 0.1, False, False + ) + hidden_states_16 = None + add_5 = hidden_states_17 + hidden_states_15 + hidden_states_17 = hidden_states_15 = None + hidden_states_18 = torch.nn.functional.layer_norm( + add_5, + (1024,), + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_5 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_19 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_20 = torch._C._nn.gelu(hidden_states_19) + hidden_states_19 = None + hidden_states_21 = torch._C._nn.linear( + hidden_states_20, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_20 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_22 = torch.nn.functional.dropout( + hidden_states_21, 0.1, False, False + ) + hidden_states_21 = None + add_6 = hidden_states_22 + hidden_states_18 + hidden_states_22 = hidden_states_18 = None + hidden_states_23 = torch.nn.functional.layer_norm( + add_6, + (1024,), + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_6 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_18 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_9 = linear_18.view(1, -1, 16, 64) + linear_18 = None + query_layer_3 = view_9.transpose(1, 2) + view_9 = None + linear_19 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_10 = linear_19.view(1, -1, 16, 64) + linear_19 = None + key_layer_3 = view_10.transpose(1, 2) + view_10 = None + linear_20 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_11 = linear_20.view(1, -1, 16, 64) + linear_20 = None + value_layer_3 = view_11.transpose(1, 2) + view_11 = None + attn_output_9 = torch._C._nn.scaled_dot_product_attention( + query_layer_3, + key_layer_3, + value_layer_3, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_3 = key_layer_3 = value_layer_3 = None + attn_output_10 = attn_output_9.transpose(1, 2) + attn_output_9 = None + attn_output_11 = attn_output_10.reshape(1, 11, 1024) + attn_output_10 = None + hidden_states_24 = torch._C._nn.linear( + attn_output_11, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_11 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_25 = torch.nn.functional.dropout( + hidden_states_24, 0.1, False, False + ) + hidden_states_24 = None + add_7 = hidden_states_25 + hidden_states_23 + hidden_states_25 = hidden_states_23 = None + hidden_states_26 = torch.nn.functional.layer_norm( + add_7, + (1024,), + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_7 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_27 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_28 = torch._C._nn.gelu(hidden_states_27) + hidden_states_27 = None + hidden_states_29 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_28 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_30 = torch.nn.functional.dropout( + hidden_states_29, 0.1, False, False + ) + hidden_states_29 = None + add_8 = hidden_states_30 + hidden_states_26 + hidden_states_30 = hidden_states_26 = None + hidden_states_31 = torch.nn.functional.layer_norm( + add_8, + (1024,), + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_8 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_24 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_12 = linear_24.view(1, -1, 16, 64) + linear_24 = None + query_layer_4 = view_12.transpose(1, 2) + view_12 = None + linear_25 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_13 = linear_25.view(1, -1, 16, 64) + linear_25 = None + key_layer_4 = view_13.transpose(1, 2) + view_13 = None + linear_26 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_14 = linear_26.view(1, -1, 16, 64) + linear_26 = None + value_layer_4 = view_14.transpose(1, 2) + view_14 = None + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_layer_4, + key_layer_4, + value_layer_4, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_4 = key_layer_4 = value_layer_4 = None + attn_output_13 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_14 = attn_output_13.reshape(1, 11, 1024) + attn_output_13 = None + hidden_states_32 = torch._C._nn.linear( + attn_output_14, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_14 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_33 = torch.nn.functional.dropout( + hidden_states_32, 0.1, False, False + ) + hidden_states_32 = None + add_9 = hidden_states_33 + hidden_states_31 + hidden_states_33 = hidden_states_31 = None + hidden_states_34 = torch.nn.functional.layer_norm( + add_9, + (1024,), + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_9 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_35 = torch._C._nn.linear( + hidden_states_34, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_36 = torch._C._nn.gelu(hidden_states_35) + hidden_states_35 = None + hidden_states_37 = torch._C._nn.linear( + hidden_states_36, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_36 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_38 = torch.nn.functional.dropout( + hidden_states_37, 0.1, False, False + ) + hidden_states_37 = None + add_10 = hidden_states_38 + hidden_states_34 + hidden_states_38 = hidden_states_34 = None + hidden_states_39 = torch.nn.functional.layer_norm( + add_10, + (1024,), + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_10 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_30 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_15 = linear_30.view(1, -1, 16, 64) + linear_30 = None + query_layer_5 = view_15.transpose(1, 2) + view_15 = None + linear_31 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_16 = linear_31.view(1, -1, 16, 64) + linear_31 = None + key_layer_5 = view_16.transpose(1, 2) + view_16 = None + linear_32 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_17 = linear_32.view(1, -1, 16, 64) + linear_32 = None + value_layer_5 = view_17.transpose(1, 2) + view_17 = None + attn_output_15 = torch._C._nn.scaled_dot_product_attention( + query_layer_5, + key_layer_5, + value_layer_5, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_5 = key_layer_5 = value_layer_5 = None + attn_output_16 = attn_output_15.transpose(1, 2) + attn_output_15 = None + attn_output_17 = attn_output_16.reshape(1, 11, 1024) + attn_output_16 = None + hidden_states_40 = torch._C._nn.linear( + attn_output_17, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_17 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_41 = torch.nn.functional.dropout( + hidden_states_40, 0.1, False, False + ) + hidden_states_40 = None + add_11 = hidden_states_41 + hidden_states_39 + hidden_states_41 = hidden_states_39 = None + hidden_states_42 = torch.nn.functional.layer_norm( + add_11, + (1024,), + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_11 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_43 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_44 = torch._C._nn.gelu(hidden_states_43) + hidden_states_43 = None + hidden_states_45 = torch._C._nn.linear( + hidden_states_44, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_44 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_46 = torch.nn.functional.dropout( + hidden_states_45, 0.1, False, False + ) + hidden_states_45 = None + add_12 = hidden_states_46 + hidden_states_42 + hidden_states_46 = hidden_states_42 = None + hidden_states_47 = torch.nn.functional.layer_norm( + add_12, + (1024,), + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_12 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_36 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_18 = linear_36.view(1, -1, 16, 64) + linear_36 = None + query_layer_6 = view_18.transpose(1, 2) + view_18 = None + linear_37 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_19 = linear_37.view(1, -1, 16, 64) + linear_37 = None + key_layer_6 = view_19.transpose(1, 2) + view_19 = None + linear_38 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_20 = linear_38.view(1, -1, 16, 64) + linear_38 = None + value_layer_6 = view_20.transpose(1, 2) + view_20 = None + attn_output_18 = torch._C._nn.scaled_dot_product_attention( + query_layer_6, + key_layer_6, + value_layer_6, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_6 = key_layer_6 = value_layer_6 = None + attn_output_19 = attn_output_18.transpose(1, 2) + attn_output_18 = None + attn_output_20 = attn_output_19.reshape(1, 11, 1024) + attn_output_19 = None + hidden_states_48 = torch._C._nn.linear( + attn_output_20, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_20 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_49 = torch.nn.functional.dropout( + hidden_states_48, 0.1, False, False + ) + hidden_states_48 = None + add_13 = hidden_states_49 + hidden_states_47 + hidden_states_49 = hidden_states_47 = None + hidden_states_50 = torch.nn.functional.layer_norm( + add_13, + (1024,), + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_13 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_51 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_52 = torch._C._nn.gelu(hidden_states_51) + hidden_states_51 = None + hidden_states_53 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_52 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_54 = torch.nn.functional.dropout( + hidden_states_53, 0.1, False, False + ) + hidden_states_53 = None + add_14 = hidden_states_54 + hidden_states_50 + hidden_states_54 = hidden_states_50 = None + hidden_states_55 = torch.nn.functional.layer_norm( + add_14, + (1024,), + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_14 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_42 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_21 = linear_42.view(1, -1, 16, 64) + linear_42 = None + query_layer_7 = view_21.transpose(1, 2) + view_21 = None + linear_43 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_22 = linear_43.view(1, -1, 16, 64) + linear_43 = None + key_layer_7 = view_22.transpose(1, 2) + view_22 = None + linear_44 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_23 = linear_44.view(1, -1, 16, 64) + linear_44 = None + value_layer_7 = view_23.transpose(1, 2) + view_23 = None + attn_output_21 = torch._C._nn.scaled_dot_product_attention( + query_layer_7, + key_layer_7, + value_layer_7, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_7 = key_layer_7 = value_layer_7 = None + attn_output_22 = attn_output_21.transpose(1, 2) + attn_output_21 = None + attn_output_23 = attn_output_22.reshape(1, 11, 1024) + attn_output_22 = None + hidden_states_56 = torch._C._nn.linear( + attn_output_23, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_23 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_57 = torch.nn.functional.dropout( + hidden_states_56, 0.1, False, False + ) + hidden_states_56 = None + add_15 = hidden_states_57 + hidden_states_55 + hidden_states_57 = hidden_states_55 = None + hidden_states_58 = torch.nn.functional.layer_norm( + add_15, + (1024,), + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_15 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_59 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_60 = torch._C._nn.gelu(hidden_states_59) + hidden_states_59 = None + hidden_states_61 = torch._C._nn.linear( + hidden_states_60, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_60 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_62 = torch.nn.functional.dropout( + hidden_states_61, 0.1, False, False + ) + hidden_states_61 = None + add_16 = hidden_states_62 + hidden_states_58 + hidden_states_62 = hidden_states_58 = None + hidden_states_63 = torch.nn.functional.layer_norm( + add_16, + (1024,), + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_16 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_48 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_24 = linear_48.view(1, -1, 16, 64) + linear_48 = None + query_layer_8 = view_24.transpose(1, 2) + view_24 = None + linear_49 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_25 = linear_49.view(1, -1, 16, 64) + linear_49 = None + key_layer_8 = view_25.transpose(1, 2) + view_25 = None + linear_50 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_26 = linear_50.view(1, -1, 16, 64) + linear_50 = None + value_layer_8 = view_26.transpose(1, 2) + view_26 = None + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_layer_8, + key_layer_8, + value_layer_8, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_8 = key_layer_8 = value_layer_8 = None + attn_output_25 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_26 = attn_output_25.reshape(1, 11, 1024) + attn_output_25 = None + hidden_states_64 = torch._C._nn.linear( + attn_output_26, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_26 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_65 = torch.nn.functional.dropout( + hidden_states_64, 0.1, False, False + ) + hidden_states_64 = None + add_17 = hidden_states_65 + hidden_states_63 + hidden_states_65 = hidden_states_63 = None + hidden_states_66 = torch.nn.functional.layer_norm( + add_17, + (1024,), + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_17 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_67 = torch._C._nn.linear( + hidden_states_66, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_68 = torch._C._nn.gelu(hidden_states_67) + hidden_states_67 = None + hidden_states_69 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_68 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_70 = torch.nn.functional.dropout( + hidden_states_69, 0.1, False, False + ) + hidden_states_69 = None + add_18 = hidden_states_70 + hidden_states_66 + hidden_states_70 = hidden_states_66 = None + hidden_states_71 = torch.nn.functional.layer_norm( + add_18, + (1024,), + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_18 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_54 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_27 = linear_54.view(1, -1, 16, 64) + linear_54 = None + query_layer_9 = view_27.transpose(1, 2) + view_27 = None + linear_55 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_28 = linear_55.view(1, -1, 16, 64) + linear_55 = None + key_layer_9 = view_28.transpose(1, 2) + view_28 = None + linear_56 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_29 = linear_56.view(1, -1, 16, 64) + linear_56 = None + value_layer_9 = view_29.transpose(1, 2) + view_29 = None + attn_output_27 = torch._C._nn.scaled_dot_product_attention( + query_layer_9, + key_layer_9, + value_layer_9, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_9 = key_layer_9 = value_layer_9 = None + attn_output_28 = attn_output_27.transpose(1, 2) + attn_output_27 = None + attn_output_29 = attn_output_28.reshape(1, 11, 1024) + attn_output_28 = None + hidden_states_72 = torch._C._nn.linear( + attn_output_29, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_29 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_73 = torch.nn.functional.dropout( + hidden_states_72, 0.1, False, False + ) + hidden_states_72 = None + add_19 = hidden_states_73 + hidden_states_71 + hidden_states_73 = hidden_states_71 = None + hidden_states_74 = torch.nn.functional.layer_norm( + add_19, + (1024,), + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_19 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_75 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_76 = torch._C._nn.gelu(hidden_states_75) + hidden_states_75 = None + hidden_states_77 = torch._C._nn.linear( + hidden_states_76, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_76 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_78 = torch.nn.functional.dropout( + hidden_states_77, 0.1, False, False + ) + hidden_states_77 = None + add_20 = hidden_states_78 + hidden_states_74 + hidden_states_78 = hidden_states_74 = None + hidden_states_79 = torch.nn.functional.layer_norm( + add_20, + (1024,), + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_20 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_60 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_30 = linear_60.view(1, -1, 16, 64) + linear_60 = None + query_layer_10 = view_30.transpose(1, 2) + view_30 = None + linear_61 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_31 = linear_61.view(1, -1, 16, 64) + linear_61 = None + key_layer_10 = view_31.transpose(1, 2) + view_31 = None + linear_62 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_32 = linear_62.view(1, -1, 16, 64) + linear_62 = None + value_layer_10 = view_32.transpose(1, 2) + view_32 = None + attn_output_30 = torch._C._nn.scaled_dot_product_attention( + query_layer_10, + key_layer_10, + value_layer_10, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_10 = key_layer_10 = value_layer_10 = None + attn_output_31 = attn_output_30.transpose(1, 2) + attn_output_30 = None + attn_output_32 = attn_output_31.reshape(1, 11, 1024) + attn_output_31 = None + hidden_states_80 = torch._C._nn.linear( + attn_output_32, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_32 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_81 = torch.nn.functional.dropout( + hidden_states_80, 0.1, False, False + ) + hidden_states_80 = None + add_21 = hidden_states_81 + hidden_states_79 + hidden_states_81 = hidden_states_79 = None + hidden_states_82 = torch.nn.functional.layer_norm( + add_21, + (1024,), + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_21 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_83 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_84 = torch._C._nn.gelu(hidden_states_83) + hidden_states_83 = None + hidden_states_85 = torch._C._nn.linear( + hidden_states_84, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_84 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_86 = torch.nn.functional.dropout( + hidden_states_85, 0.1, False, False + ) + hidden_states_85 = None + add_22 = hidden_states_86 + hidden_states_82 + hidden_states_86 = hidden_states_82 = None + hidden_states_87 = torch.nn.functional.layer_norm( + add_22, + (1024,), + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_22 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_66 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_33 = linear_66.view(1, -1, 16, 64) + linear_66 = None + query_layer_11 = view_33.transpose(1, 2) + view_33 = None + linear_67 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_34 = linear_67.view(1, -1, 16, 64) + linear_67 = None + key_layer_11 = view_34.transpose(1, 2) + view_34 = None + linear_68 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_35 = linear_68.view(1, -1, 16, 64) + linear_68 = None + value_layer_11 = view_35.transpose(1, 2) + view_35 = None + attn_output_33 = torch._C._nn.scaled_dot_product_attention( + query_layer_11, + key_layer_11, + value_layer_11, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_11 = key_layer_11 = value_layer_11 = None + attn_output_34 = attn_output_33.transpose(1, 2) + attn_output_33 = None + attn_output_35 = attn_output_34.reshape(1, 11, 1024) + attn_output_34 = None + hidden_states_88 = torch._C._nn.linear( + attn_output_35, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_35 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_89 = torch.nn.functional.dropout( + hidden_states_88, 0.1, False, False + ) + hidden_states_88 = None + add_23 = hidden_states_89 + hidden_states_87 + hidden_states_89 = hidden_states_87 = None + hidden_states_90 = torch.nn.functional.layer_norm( + add_23, + (1024,), + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_23 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_91 = torch._C._nn.linear( + hidden_states_90, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_92 = torch._C._nn.gelu(hidden_states_91) + hidden_states_91 = None + hidden_states_93 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_92 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_94 = torch.nn.functional.dropout( + hidden_states_93, 0.1, False, False + ) + hidden_states_93 = None + add_24 = hidden_states_94 + hidden_states_90 + hidden_states_94 = hidden_states_90 = None + hidden_states_95 = torch.nn.functional.layer_norm( + add_24, + (1024,), + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_24 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_72 = torch._C._nn.linear( + hidden_states_95, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_36 = linear_72.view(1, -1, 16, 64) + linear_72 = None + query_layer_12 = view_36.transpose(1, 2) + view_36 = None + linear_73 = torch._C._nn.linear( + hidden_states_95, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_37 = linear_73.view(1, -1, 16, 64) + linear_73 = None + key_layer_12 = view_37.transpose(1, 2) + view_37 = None + linear_74 = torch._C._nn.linear( + hidden_states_95, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_38 = linear_74.view(1, -1, 16, 64) + linear_74 = None + value_layer_12 = view_38.transpose(1, 2) + view_38 = None + attn_output_36 = torch._C._nn.scaled_dot_product_attention( + query_layer_12, + key_layer_12, + value_layer_12, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_12 = key_layer_12 = value_layer_12 = None + attn_output_37 = attn_output_36.transpose(1, 2) + attn_output_36 = None + attn_output_38 = attn_output_37.reshape(1, 11, 1024) + attn_output_37 = None + hidden_states_96 = torch._C._nn.linear( + attn_output_38, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_38 = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_97 = torch.nn.functional.dropout( + hidden_states_96, 0.1, False, False + ) + hidden_states_96 = None + add_25 = hidden_states_97 + hidden_states_95 + hidden_states_97 = hidden_states_95 = None + hidden_states_98 = torch.nn.functional.layer_norm( + add_25, + (1024,), + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_25 = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_99 = torch._C._nn.linear( + hidden_states_98, + l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_100 = torch._C._nn.gelu(hidden_states_99) + hidden_states_99 = None + hidden_states_101 = torch._C._nn.linear( + hidden_states_100, + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_100 = l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_102 = torch.nn.functional.dropout( + hidden_states_101, 0.1, False, False + ) + hidden_states_101 = None + add_26 = hidden_states_102 + hidden_states_98 + hidden_states_102 = hidden_states_98 = None + hidden_states_103 = torch.nn.functional.layer_norm( + add_26, + (1024,), + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_26 = l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_12_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_78 = torch._C._nn.linear( + hidden_states_103, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_39 = linear_78.view(1, -1, 16, 64) + linear_78 = None + query_layer_13 = view_39.transpose(1, 2) + view_39 = None + linear_79 = torch._C._nn.linear( + hidden_states_103, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_40 = linear_79.view(1, -1, 16, 64) + linear_79 = None + key_layer_13 = view_40.transpose(1, 2) + view_40 = None + linear_80 = torch._C._nn.linear( + hidden_states_103, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_41 = linear_80.view(1, -1, 16, 64) + linear_80 = None + value_layer_13 = view_41.transpose(1, 2) + view_41 = None + attn_output_39 = torch._C._nn.scaled_dot_product_attention( + query_layer_13, + key_layer_13, + value_layer_13, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_13 = key_layer_13 = value_layer_13 = None + attn_output_40 = attn_output_39.transpose(1, 2) + attn_output_39 = None + attn_output_41 = attn_output_40.reshape(1, 11, 1024) + attn_output_40 = None + hidden_states_104 = torch._C._nn.linear( + attn_output_41, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_41 = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_105 = torch.nn.functional.dropout( + hidden_states_104, 0.1, False, False + ) + hidden_states_104 = None + add_27 = hidden_states_105 + hidden_states_103 + hidden_states_105 = hidden_states_103 = None + hidden_states_106 = torch.nn.functional.layer_norm( + add_27, + (1024,), + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_27 = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_107 = torch._C._nn.linear( + hidden_states_106, + l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_108 = torch._C._nn.gelu(hidden_states_107) + hidden_states_107 = None + hidden_states_109 = torch._C._nn.linear( + hidden_states_108, + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_108 = l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_110 = torch.nn.functional.dropout( + hidden_states_109, 0.1, False, False + ) + hidden_states_109 = None + add_28 = hidden_states_110 + hidden_states_106 + hidden_states_110 = hidden_states_106 = None + hidden_states_111 = torch.nn.functional.layer_norm( + add_28, + (1024,), + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_28 = l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_13_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_84 = torch._C._nn.linear( + hidden_states_111, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_42 = linear_84.view(1, -1, 16, 64) + linear_84 = None + query_layer_14 = view_42.transpose(1, 2) + view_42 = None + linear_85 = torch._C._nn.linear( + hidden_states_111, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_43 = linear_85.view(1, -1, 16, 64) + linear_85 = None + key_layer_14 = view_43.transpose(1, 2) + view_43 = None + linear_86 = torch._C._nn.linear( + hidden_states_111, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_44 = linear_86.view(1, -1, 16, 64) + linear_86 = None + value_layer_14 = view_44.transpose(1, 2) + view_44 = None + attn_output_42 = torch._C._nn.scaled_dot_product_attention( + query_layer_14, + key_layer_14, + value_layer_14, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_14 = key_layer_14 = value_layer_14 = None + attn_output_43 = attn_output_42.transpose(1, 2) + attn_output_42 = None + attn_output_44 = attn_output_43.reshape(1, 11, 1024) + attn_output_43 = None + hidden_states_112 = torch._C._nn.linear( + attn_output_44, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_44 = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_113 = torch.nn.functional.dropout( + hidden_states_112, 0.1, False, False + ) + hidden_states_112 = None + add_29 = hidden_states_113 + hidden_states_111 + hidden_states_113 = hidden_states_111 = None + hidden_states_114 = torch.nn.functional.layer_norm( + add_29, + (1024,), + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_29 = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_115 = torch._C._nn.linear( + hidden_states_114, + l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_116 = torch._C._nn.gelu(hidden_states_115) + hidden_states_115 = None + hidden_states_117 = torch._C._nn.linear( + hidden_states_116, + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_116 = l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_118 = torch.nn.functional.dropout( + hidden_states_117, 0.1, False, False + ) + hidden_states_117 = None + add_30 = hidden_states_118 + hidden_states_114 + hidden_states_118 = hidden_states_114 = None + hidden_states_119 = torch.nn.functional.layer_norm( + add_30, + (1024,), + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_30 = l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_14_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_90 = torch._C._nn.linear( + hidden_states_119, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_45 = linear_90.view(1, -1, 16, 64) + linear_90 = None + query_layer_15 = view_45.transpose(1, 2) + view_45 = None + linear_91 = torch._C._nn.linear( + hidden_states_119, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_46 = linear_91.view(1, -1, 16, 64) + linear_91 = None + key_layer_15 = view_46.transpose(1, 2) + view_46 = None + linear_92 = torch._C._nn.linear( + hidden_states_119, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_47 = linear_92.view(1, -1, 16, 64) + linear_92 = None + value_layer_15 = view_47.transpose(1, 2) + view_47 = None + attn_output_45 = torch._C._nn.scaled_dot_product_attention( + query_layer_15, + key_layer_15, + value_layer_15, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_15 = key_layer_15 = value_layer_15 = None + attn_output_46 = attn_output_45.transpose(1, 2) + attn_output_45 = None + attn_output_47 = attn_output_46.reshape(1, 11, 1024) + attn_output_46 = None + hidden_states_120 = torch._C._nn.linear( + attn_output_47, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_47 = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_121 = torch.nn.functional.dropout( + hidden_states_120, 0.1, False, False + ) + hidden_states_120 = None + add_31 = hidden_states_121 + hidden_states_119 + hidden_states_121 = hidden_states_119 = None + hidden_states_122 = torch.nn.functional.layer_norm( + add_31, + (1024,), + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_31 = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_123 = torch._C._nn.linear( + hidden_states_122, + l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_124 = torch._C._nn.gelu(hidden_states_123) + hidden_states_123 = None + hidden_states_125 = torch._C._nn.linear( + hidden_states_124, + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_124 = l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_126 = torch.nn.functional.dropout( + hidden_states_125, 0.1, False, False + ) + hidden_states_125 = None + add_32 = hidden_states_126 + hidden_states_122 + hidden_states_126 = hidden_states_122 = None + hidden_states_127 = torch.nn.functional.layer_norm( + add_32, + (1024,), + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_32 = l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_15_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_96 = torch._C._nn.linear( + hidden_states_127, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_48 = linear_96.view(1, -1, 16, 64) + linear_96 = None + query_layer_16 = view_48.transpose(1, 2) + view_48 = None + linear_97 = torch._C._nn.linear( + hidden_states_127, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_49 = linear_97.view(1, -1, 16, 64) + linear_97 = None + key_layer_16 = view_49.transpose(1, 2) + view_49 = None + linear_98 = torch._C._nn.linear( + hidden_states_127, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_50 = linear_98.view(1, -1, 16, 64) + linear_98 = None + value_layer_16 = view_50.transpose(1, 2) + view_50 = None + attn_output_48 = torch._C._nn.scaled_dot_product_attention( + query_layer_16, + key_layer_16, + value_layer_16, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_16 = key_layer_16 = value_layer_16 = None + attn_output_49 = attn_output_48.transpose(1, 2) + attn_output_48 = None + attn_output_50 = attn_output_49.reshape(1, 11, 1024) + attn_output_49 = None + hidden_states_128 = torch._C._nn.linear( + attn_output_50, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_50 = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_129 = torch.nn.functional.dropout( + hidden_states_128, 0.1, False, False + ) + hidden_states_128 = None + add_33 = hidden_states_129 + hidden_states_127 + hidden_states_129 = hidden_states_127 = None + hidden_states_130 = torch.nn.functional.layer_norm( + add_33, + (1024,), + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_33 = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_131 = torch._C._nn.linear( + hidden_states_130, + l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_132 = torch._C._nn.gelu(hidden_states_131) + hidden_states_131 = None + hidden_states_133 = torch._C._nn.linear( + hidden_states_132, + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_132 = l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_134 = torch.nn.functional.dropout( + hidden_states_133, 0.1, False, False + ) + hidden_states_133 = None + add_34 = hidden_states_134 + hidden_states_130 + hidden_states_134 = hidden_states_130 = None + hidden_states_135 = torch.nn.functional.layer_norm( + add_34, + (1024,), + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_34 = l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_16_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_102 = torch._C._nn.linear( + hidden_states_135, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_51 = linear_102.view(1, -1, 16, 64) + linear_102 = None + query_layer_17 = view_51.transpose(1, 2) + view_51 = None + linear_103 = torch._C._nn.linear( + hidden_states_135, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_52 = linear_103.view(1, -1, 16, 64) + linear_103 = None + key_layer_17 = view_52.transpose(1, 2) + view_52 = None + linear_104 = torch._C._nn.linear( + hidden_states_135, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_53 = linear_104.view(1, -1, 16, 64) + linear_104 = None + value_layer_17 = view_53.transpose(1, 2) + view_53 = None + attn_output_51 = torch._C._nn.scaled_dot_product_attention( + query_layer_17, + key_layer_17, + value_layer_17, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_17 = key_layer_17 = value_layer_17 = None + attn_output_52 = attn_output_51.transpose(1, 2) + attn_output_51 = None + attn_output_53 = attn_output_52.reshape(1, 11, 1024) + attn_output_52 = None + hidden_states_136 = torch._C._nn.linear( + attn_output_53, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_53 = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_137 = torch.nn.functional.dropout( + hidden_states_136, 0.1, False, False + ) + hidden_states_136 = None + add_35 = hidden_states_137 + hidden_states_135 + hidden_states_137 = hidden_states_135 = None + hidden_states_138 = torch.nn.functional.layer_norm( + add_35, + (1024,), + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_35 = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_139 = torch._C._nn.linear( + hidden_states_138, + l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_140 = torch._C._nn.gelu(hidden_states_139) + hidden_states_139 = None + hidden_states_141 = torch._C._nn.linear( + hidden_states_140, + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_140 = l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_142 = torch.nn.functional.dropout( + hidden_states_141, 0.1, False, False + ) + hidden_states_141 = None + add_36 = hidden_states_142 + hidden_states_138 + hidden_states_142 = hidden_states_138 = None + hidden_states_143 = torch.nn.functional.layer_norm( + add_36, + (1024,), + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_36 = l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_17_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_108 = torch._C._nn.linear( + hidden_states_143, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_54 = linear_108.view(1, -1, 16, 64) + linear_108 = None + query_layer_18 = view_54.transpose(1, 2) + view_54 = None + linear_109 = torch._C._nn.linear( + hidden_states_143, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_55 = linear_109.view(1, -1, 16, 64) + linear_109 = None + key_layer_18 = view_55.transpose(1, 2) + view_55 = None + linear_110 = torch._C._nn.linear( + hidden_states_143, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_56 = linear_110.view(1, -1, 16, 64) + linear_110 = None + value_layer_18 = view_56.transpose(1, 2) + view_56 = None + attn_output_54 = torch._C._nn.scaled_dot_product_attention( + query_layer_18, + key_layer_18, + value_layer_18, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_18 = key_layer_18 = value_layer_18 = None + attn_output_55 = attn_output_54.transpose(1, 2) + attn_output_54 = None + attn_output_56 = attn_output_55.reshape(1, 11, 1024) + attn_output_55 = None + hidden_states_144 = torch._C._nn.linear( + attn_output_56, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_56 = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_145 = torch.nn.functional.dropout( + hidden_states_144, 0.1, False, False + ) + hidden_states_144 = None + add_37 = hidden_states_145 + hidden_states_143 + hidden_states_145 = hidden_states_143 = None + hidden_states_146 = torch.nn.functional.layer_norm( + add_37, + (1024,), + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_37 = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_147 = torch._C._nn.linear( + hidden_states_146, + l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_148 = torch._C._nn.gelu(hidden_states_147) + hidden_states_147 = None + hidden_states_149 = torch._C._nn.linear( + hidden_states_148, + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_148 = l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_150 = torch.nn.functional.dropout( + hidden_states_149, 0.1, False, False + ) + hidden_states_149 = None + add_38 = hidden_states_150 + hidden_states_146 + hidden_states_150 = hidden_states_146 = None + hidden_states_151 = torch.nn.functional.layer_norm( + add_38, + (1024,), + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_38 = l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_18_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_114 = torch._C._nn.linear( + hidden_states_151, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_57 = linear_114.view(1, -1, 16, 64) + linear_114 = None + query_layer_19 = view_57.transpose(1, 2) + view_57 = None + linear_115 = torch._C._nn.linear( + hidden_states_151, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_58 = linear_115.view(1, -1, 16, 64) + linear_115 = None + key_layer_19 = view_58.transpose(1, 2) + view_58 = None + linear_116 = torch._C._nn.linear( + hidden_states_151, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_59 = linear_116.view(1, -1, 16, 64) + linear_116 = None + value_layer_19 = view_59.transpose(1, 2) + view_59 = None + attn_output_57 = torch._C._nn.scaled_dot_product_attention( + query_layer_19, + key_layer_19, + value_layer_19, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_19 = key_layer_19 = value_layer_19 = None + attn_output_58 = attn_output_57.transpose(1, 2) + attn_output_57 = None + attn_output_59 = attn_output_58.reshape(1, 11, 1024) + attn_output_58 = None + hidden_states_152 = torch._C._nn.linear( + attn_output_59, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_59 = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_153 = torch.nn.functional.dropout( + hidden_states_152, 0.1, False, False + ) + hidden_states_152 = None + add_39 = hidden_states_153 + hidden_states_151 + hidden_states_153 = hidden_states_151 = None + hidden_states_154 = torch.nn.functional.layer_norm( + add_39, + (1024,), + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_39 = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_155 = torch._C._nn.linear( + hidden_states_154, + l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_156 = torch._C._nn.gelu(hidden_states_155) + hidden_states_155 = None + hidden_states_157 = torch._C._nn.linear( + hidden_states_156, + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_156 = l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_158 = torch.nn.functional.dropout( + hidden_states_157, 0.1, False, False + ) + hidden_states_157 = None + add_40 = hidden_states_158 + hidden_states_154 + hidden_states_158 = hidden_states_154 = None + hidden_states_159 = torch.nn.functional.layer_norm( + add_40, + (1024,), + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_40 = l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_19_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_120 = torch._C._nn.linear( + hidden_states_159, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_60 = linear_120.view(1, -1, 16, 64) + linear_120 = None + query_layer_20 = view_60.transpose(1, 2) + view_60 = None + linear_121 = torch._C._nn.linear( + hidden_states_159, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_61 = linear_121.view(1, -1, 16, 64) + linear_121 = None + key_layer_20 = view_61.transpose(1, 2) + view_61 = None + linear_122 = torch._C._nn.linear( + hidden_states_159, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_62 = linear_122.view(1, -1, 16, 64) + linear_122 = None + value_layer_20 = view_62.transpose(1, 2) + view_62 = None + attn_output_60 = torch._C._nn.scaled_dot_product_attention( + query_layer_20, + key_layer_20, + value_layer_20, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_20 = key_layer_20 = value_layer_20 = None + attn_output_61 = attn_output_60.transpose(1, 2) + attn_output_60 = None + attn_output_62 = attn_output_61.reshape(1, 11, 1024) + attn_output_61 = None + hidden_states_160 = torch._C._nn.linear( + attn_output_62, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_62 = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_161 = torch.nn.functional.dropout( + hidden_states_160, 0.1, False, False + ) + hidden_states_160 = None + add_41 = hidden_states_161 + hidden_states_159 + hidden_states_161 = hidden_states_159 = None + hidden_states_162 = torch.nn.functional.layer_norm( + add_41, + (1024,), + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_41 = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_163 = torch._C._nn.linear( + hidden_states_162, + l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_164 = torch._C._nn.gelu(hidden_states_163) + hidden_states_163 = None + hidden_states_165 = torch._C._nn.linear( + hidden_states_164, + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_164 = l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_166 = torch.nn.functional.dropout( + hidden_states_165, 0.1, False, False + ) + hidden_states_165 = None + add_42 = hidden_states_166 + hidden_states_162 + hidden_states_166 = hidden_states_162 = None + hidden_states_167 = torch.nn.functional.layer_norm( + add_42, + (1024,), + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_42 = l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_20_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_126 = torch._C._nn.linear( + hidden_states_167, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_63 = linear_126.view(1, -1, 16, 64) + linear_126 = None + query_layer_21 = view_63.transpose(1, 2) + view_63 = None + linear_127 = torch._C._nn.linear( + hidden_states_167, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_64 = linear_127.view(1, -1, 16, 64) + linear_127 = None + key_layer_21 = view_64.transpose(1, 2) + view_64 = None + linear_128 = torch._C._nn.linear( + hidden_states_167, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_65 = linear_128.view(1, -1, 16, 64) + linear_128 = None + value_layer_21 = view_65.transpose(1, 2) + view_65 = None + attn_output_63 = torch._C._nn.scaled_dot_product_attention( + query_layer_21, + key_layer_21, + value_layer_21, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_21 = key_layer_21 = value_layer_21 = None + attn_output_64 = attn_output_63.transpose(1, 2) + attn_output_63 = None + attn_output_65 = attn_output_64.reshape(1, 11, 1024) + attn_output_64 = None + hidden_states_168 = torch._C._nn.linear( + attn_output_65, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_65 = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_169 = torch.nn.functional.dropout( + hidden_states_168, 0.1, False, False + ) + hidden_states_168 = None + add_43 = hidden_states_169 + hidden_states_167 + hidden_states_169 = hidden_states_167 = None + hidden_states_170 = torch.nn.functional.layer_norm( + add_43, + (1024,), + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_43 = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_171 = torch._C._nn.linear( + hidden_states_170, + l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_172 = torch._C._nn.gelu(hidden_states_171) + hidden_states_171 = None + hidden_states_173 = torch._C._nn.linear( + hidden_states_172, + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_172 = l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_174 = torch.nn.functional.dropout( + hidden_states_173, 0.1, False, False + ) + hidden_states_173 = None + add_44 = hidden_states_174 + hidden_states_170 + hidden_states_174 = hidden_states_170 = None + hidden_states_175 = torch.nn.functional.layer_norm( + add_44, + (1024,), + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_44 = l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_21_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_132 = torch._C._nn.linear( + hidden_states_175, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_66 = linear_132.view(1, -1, 16, 64) + linear_132 = None + query_layer_22 = view_66.transpose(1, 2) + view_66 = None + linear_133 = torch._C._nn.linear( + hidden_states_175, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_67 = linear_133.view(1, -1, 16, 64) + linear_133 = None + key_layer_22 = view_67.transpose(1, 2) + view_67 = None + linear_134 = torch._C._nn.linear( + hidden_states_175, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_68 = linear_134.view(1, -1, 16, 64) + linear_134 = None + value_layer_22 = view_68.transpose(1, 2) + view_68 = None + attn_output_66 = torch._C._nn.scaled_dot_product_attention( + query_layer_22, + key_layer_22, + value_layer_22, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_22 = key_layer_22 = value_layer_22 = None + attn_output_67 = attn_output_66.transpose(1, 2) + attn_output_66 = None + attn_output_68 = attn_output_67.reshape(1, 11, 1024) + attn_output_67 = None + hidden_states_176 = torch._C._nn.linear( + attn_output_68, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_68 = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_177 = torch.nn.functional.dropout( + hidden_states_176, 0.1, False, False + ) + hidden_states_176 = None + add_45 = hidden_states_177 + hidden_states_175 + hidden_states_177 = hidden_states_175 = None + hidden_states_178 = torch.nn.functional.layer_norm( + add_45, + (1024,), + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_45 = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_179 = torch._C._nn.linear( + hidden_states_178, + l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_180 = torch._C._nn.gelu(hidden_states_179) + hidden_states_179 = None + hidden_states_181 = torch._C._nn.linear( + hidden_states_180, + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_180 = l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_182 = torch.nn.functional.dropout( + hidden_states_181, 0.1, False, False + ) + hidden_states_181 = None + add_46 = hidden_states_182 + hidden_states_178 + hidden_states_182 = hidden_states_178 = None + hidden_states_183 = torch.nn.functional.layer_norm( + add_46, + (1024,), + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_46 = l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_22_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_138 = torch._C._nn.linear( + hidden_states_183, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_69 = linear_138.view(1, -1, 16, 64) + linear_138 = None + query_layer_23 = view_69.transpose(1, 2) + view_69 = None + linear_139 = torch._C._nn.linear( + hidden_states_183, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_70 = linear_139.view(1, -1, 16, 64) + linear_139 = None + key_layer_23 = view_70.transpose(1, 2) + view_70 = None + linear_140 = torch._C._nn.linear( + hidden_states_183, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_71 = linear_140.view(1, -1, 16, 64) + linear_140 = None + value_layer_23 = view_71.transpose(1, 2) + view_71 = None + attn_output_69 = torch._C._nn.scaled_dot_product_attention( + query_layer_23, + key_layer_23, + value_layer_23, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_23 = key_layer_23 = value_layer_23 = extended_attention_mask = None + attn_output_70 = attn_output_69.transpose(1, 2) + attn_output_69 = None + attn_output_71 = attn_output_70.reshape(1, 11, 1024) + attn_output_70 = None + hidden_states_184 = torch._C._nn.linear( + attn_output_71, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_71 = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_185 = torch.nn.functional.dropout( + hidden_states_184, 0.1, False, False + ) + hidden_states_184 = None + add_47 = hidden_states_185 + hidden_states_183 + hidden_states_185 = hidden_states_183 = None + hidden_states_186 = torch.nn.functional.layer_norm( + add_47, + (1024,), + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_47 = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_187 = torch._C._nn.linear( + hidden_states_186, + l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_188 = torch._C._nn.gelu(hidden_states_187) + hidden_states_187 = None + hidden_states_189 = torch._C._nn.linear( + hidden_states_188, + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_188 = l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_190 = torch.nn.functional.dropout( + hidden_states_189, 0.1, False, False + ) + hidden_states_189 = None + add_48 = hidden_states_190 + hidden_states_186 + hidden_states_190 = hidden_states_186 = None + hidden_states_191 = torch.nn.functional.layer_norm( + add_48, + (1024,), + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_48 = l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_23_modules_output_modules_layer_norm_parameters_bias_ = (None) + first_token_tensor = hidden_states_191[(slice(None, None, None), 0)] + pooled_output = torch._C._nn.linear( + first_token_tensor, + l_self_modules_pooler_modules_dense_parameters_weight_, + l_self_modules_pooler_modules_dense_parameters_bias_, + ) + first_token_tensor = ( + l_self_modules_pooler_modules_dense_parameters_weight_ + ) = l_self_modules_pooler_modules_dense_parameters_bias_ = None + pooled_output_1 = torch.tanh(pooled_output) + pooled_output = None + return (hidden_states_191, pooled_output_1) diff --git a/samples/transformers-auto-model/google-bert_bert-large-uncased/weight_meta.py b/samples/transformers-auto-model/google-bert_bert-large-uncased/weight_meta.py new file mode 100644 index 0000000000..e0e6b64b71 --- /dev/null +++ b/samples/transformers-auto-model/google-bert_bert-large-uncased/weight_meta.py @@ -0,0 +1,4461 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [101, 10629, 7159, 2003, 2109, 2000, 14817, 15078, 19287, 1012, 102] + + +class Program_weight_tensor_meta_L_token_type_ids_: + name = "L_token_type_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_buffers_position_ids_: + name = "L_self_modules_embeddings_buffers_position_ids_" + shape = [1, 512] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + ] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_word_embeddings_parameters_weight_" + shape = [30522, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.019 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_" + shape = [2, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_position_embeddings_parameters_weight_" + shape = [512, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.017 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.843 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.015 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.178 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.002 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.022 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.037 + std = 0.096 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.282 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.070 + std = 0.021 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.884 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.023 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.002 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.011 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.021 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.171 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.053 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.927 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.026 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.002 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.996 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.151 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.052 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.912 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.025 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.016 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.980 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.053 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.906 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.026 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.962 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.142 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.054 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.906 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.032 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.011 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.944 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.159 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.056 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.866 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.031 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.010 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.952 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.161 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.057 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.856 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.020 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.021 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.961 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.142 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.056 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.877 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.017 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.952 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.055 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.911 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.023 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.022 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.945 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.122 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.054 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.907 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.020 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.919 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.058 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.895 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.022 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.914 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.119 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.056 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.882 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.022 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.907 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.013 + std = 0.119 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.059 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.858 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_12_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.883 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.013 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.061 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.834 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_13_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.006 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.868 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.138 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.060 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.810 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_14_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.100 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.865 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.129 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.059 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.820 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_15_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.015 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.870 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.013 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.060 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.863 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_16_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.023 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.006 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.861 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.125 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.062 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.852 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_17_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.026 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.838 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.119 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.066 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.844 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_18_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.017 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.110 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.016 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.818 + std = 0.110 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.068 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.832 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_19_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.160 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.799 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.062 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.866 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_20_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.118 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.823 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.060 + std = 0.028 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.894 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_21_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.106 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.834 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.059 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.913 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_22_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.116 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.004 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.848 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.010 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_weight_" + shape = [4096, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_intermediate_modules_dense_parameters_bias_" + shape = [4096] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.056 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_weight_" + shape = [1024, 4096] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_weight_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.719 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_23_modules_output_modules_LayerNorm_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_weight_: + name = "L_self_modules_pooler_modules_dense_parameters_weight_" + shape = [1024, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.030 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_bias_: + name = "L_self_modules_pooler_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.015 + data = None diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/graph_hash.txt b/samples/transformers-auto-model/google_electra-small-discriminator/graph_hash.txt new file mode 100644 index 0000000000..56c7268ac2 --- /dev/null +++ b/samples/transformers-auto-model/google_electra-small-discriminator/graph_hash.txt @@ -0,0 +1 @@ +295e8ed5234a63374ae628854cdc1947294a2a9da2212f5ad854976604cbedf0 \ No newline at end of file diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/graph_net.json b/samples/transformers-auto-model/google_electra-small-discriminator/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/google_electra-small-discriminator/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/input_meta.py b/samples/transformers-auto-model/google_electra-small-discriminator/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/input_tensor_constraints.py b/samples/transformers-auto-model/google_electra-small-discriminator/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/model.py b/samples/transformers-auto-model/google_electra-small-discriminator/model.py new file mode 100644 index 0000000000..0de3eaa9c0 --- /dev/null +++ b/samples/transformers-auto-model/google_electra-small-discriminator/model.py @@ -0,0 +1,1705 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_attention_mask_: torch.Tensor, + L_token_type_ids_: torch.Tensor, + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_buffers_position_ids_: torch.Tensor, + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_project_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_project_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_attention_mask_ = L_attention_mask_ + l_token_type_ids_ = L_token_type_ids_ + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_buffers_position_ids_ = ( + L_self_modules_embeddings_buffers_position_ids_ + ) + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_ + ) + l_self_modules_embeddings_project_parameters_weight_ = ( + L_self_modules_embeddings_project_parameters_weight_ + ) + l_self_modules_embeddings_project_parameters_bias_ = ( + L_self_modules_embeddings_project_parameters_bias_ + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_ + extended_attention_mask = l_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_attention_mask_ = None + extended_attention_mask_1 = extended_attention_mask.to(dtype=torch.float32) + extended_attention_mask = None + sub = 1.0 - extended_attention_mask_1 + extended_attention_mask_1 = None + extended_attention_mask_2 = sub * -3.4028234663852886e38 + sub = None + position_ids = l_self_modules_embeddings_buffers_position_ids_[ + (slice(None, None, None), slice(0, 11, None)) + ] + l_self_modules_embeddings_buffers_position_ids_ = None + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_, + 0, + None, + 2.0, + False, + False, + ) + l_input_ids_ = ( + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) = None + token_type_embeddings = torch.nn.functional.embedding( + l_token_type_ids_, + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_token_type_ids_ = ( + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) = None + embeddings = inputs_embeds + token_type_embeddings + inputs_embeds = token_type_embeddings = None + position_embeddings = torch.nn.functional.embedding( + position_ids, + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = ( + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) = None + embeddings += position_embeddings + embeddings_1 = embeddings + embeddings = position_embeddings = None + embeddings_2 = torch.nn.functional.layer_norm( + embeddings_1, + (128,), + l_self_modules_embeddings_modules_layer_norm_parameters_weight_, + l_self_modules_embeddings_modules_layer_norm_parameters_bias_, + 1e-12, + ) + embeddings_1 = ( + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ + ) = l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = None + embeddings_3 = torch.nn.functional.dropout(embeddings_2, 0.1, False, False) + embeddings_2 = None + hidden_states = torch._C._nn.linear( + embeddings_3, + l_self_modules_embeddings_project_parameters_weight_, + l_self_modules_embeddings_project_parameters_bias_, + ) + embeddings_3 = ( + l_self_modules_embeddings_project_parameters_weight_ + ) = l_self_modules_embeddings_project_parameters_bias_ = None + query_layer = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view = query_layer.view(1, -1, 4, 64) + query_layer = None + query_layer_1 = view.transpose(1, 2) + view = None + key_layer = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_1 = key_layer.view(1, -1, 4, 64) + key_layer = None + key_layer_1 = view_1.transpose(1, 2) + view_1 = None + value_layer = torch._C._nn.linear( + hidden_states, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_2 = value_layer.view(1, -1, 4, 64) + value_layer = None + value_layer_1 = view_2.transpose(1, 2) + view_2 = None + transpose_3 = key_layer_1.transpose(-1, -2) + key_layer_1 = None + attention_scores = torch.matmul(query_layer_1, transpose_3) + query_layer_1 = transpose_3 = None + attention_scores_1 = attention_scores / 8.0 + attention_scores = None + attention_scores_2 = attention_scores_1 + extended_attention_mask_2 + attention_scores_1 = None + attention_probs = torch.nn.functional.softmax(attention_scores_2, dim=-1) + attention_scores_2 = None + attention_probs_1 = torch.nn.functional.dropout( + attention_probs, 0.1, False, False + ) + attention_probs = None + context_layer = torch.matmul(attention_probs_1, value_layer_1) + attention_probs_1 = value_layer_1 = None + permute = context_layer.permute(0, 2, 1, 3) + context_layer = None + context_layer_1 = permute.contiguous() + permute = None + context_layer_2 = context_layer_1.view((1, 11, 256)) + context_layer_1 = None + hidden_states_1 = torch._C._nn.linear( + context_layer_2, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_2 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_2 = torch.nn.functional.dropout( + hidden_states_1, 0.1, False, False + ) + hidden_states_1 = None + add_2 = hidden_states_2 + hidden_states + hidden_states_2 = hidden_states = None + hidden_states_3 = torch.nn.functional.layer_norm( + add_2, + (256,), + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_2 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_4 = torch._C._nn.linear( + hidden_states_3, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_5 = torch._C._nn.gelu(hidden_states_4) + hidden_states_4 = None + hidden_states_6 = torch._C._nn.linear( + hidden_states_5, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_5 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_7 = torch.nn.functional.dropout( + hidden_states_6, 0.1, False, False + ) + hidden_states_6 = None + add_3 = hidden_states_7 + hidden_states_3 + hidden_states_7 = hidden_states_3 = None + hidden_states_8 = torch.nn.functional.layer_norm( + add_3, + (256,), + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_3 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_2 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_4 = query_layer_2.view(1, -1, 4, 64) + query_layer_2 = None + query_layer_3 = view_4.transpose(1, 2) + view_4 = None + key_layer_2 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_5 = key_layer_2.view(1, -1, 4, 64) + key_layer_2 = None + key_layer_3 = view_5.transpose(1, 2) + view_5 = None + value_layer_2 = torch._C._nn.linear( + hidden_states_8, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_6 = value_layer_2.view(1, -1, 4, 64) + value_layer_2 = None + value_layer_3 = view_6.transpose(1, 2) + view_6 = None + transpose_7 = key_layer_3.transpose(-1, -2) + key_layer_3 = None + attention_scores_3 = torch.matmul(query_layer_3, transpose_7) + query_layer_3 = transpose_7 = None + attention_scores_4 = attention_scores_3 / 8.0 + attention_scores_3 = None + attention_scores_5 = attention_scores_4 + extended_attention_mask_2 + attention_scores_4 = None + attention_probs_2 = torch.nn.functional.softmax(attention_scores_5, dim=-1) + attention_scores_5 = None + attention_probs_3 = torch.nn.functional.dropout( + attention_probs_2, 0.1, False, False + ) + attention_probs_2 = None + context_layer_3 = torch.matmul(attention_probs_3, value_layer_3) + attention_probs_3 = value_layer_3 = None + permute_1 = context_layer_3.permute(0, 2, 1, 3) + context_layer_3 = None + context_layer_4 = permute_1.contiguous() + permute_1 = None + context_layer_5 = context_layer_4.view((1, 11, 256)) + context_layer_4 = None + hidden_states_9 = torch._C._nn.linear( + context_layer_5, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_10 = torch.nn.functional.dropout( + hidden_states_9, 0.1, False, False + ) + hidden_states_9 = None + add_5 = hidden_states_10 + hidden_states_8 + hidden_states_10 = hidden_states_8 = None + hidden_states_11 = torch.nn.functional.layer_norm( + add_5, + (256,), + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_12 = torch._C._nn.linear( + hidden_states_11, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_13 = torch._C._nn.gelu(hidden_states_12) + hidden_states_12 = None + hidden_states_14 = torch._C._nn.linear( + hidden_states_13, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_13 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_15 = torch.nn.functional.dropout( + hidden_states_14, 0.1, False, False + ) + hidden_states_14 = None + add_6 = hidden_states_15 + hidden_states_11 + hidden_states_15 = hidden_states_11 = None + hidden_states_16 = torch.nn.functional.layer_norm( + add_6, + (256,), + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_6 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_4 = torch._C._nn.linear( + hidden_states_16, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_8 = query_layer_4.view(1, -1, 4, 64) + query_layer_4 = None + query_layer_5 = view_8.transpose(1, 2) + view_8 = None + key_layer_4 = torch._C._nn.linear( + hidden_states_16, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_9 = key_layer_4.view(1, -1, 4, 64) + key_layer_4 = None + key_layer_5 = view_9.transpose(1, 2) + view_9 = None + value_layer_4 = torch._C._nn.linear( + hidden_states_16, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_10 = value_layer_4.view(1, -1, 4, 64) + value_layer_4 = None + value_layer_5 = view_10.transpose(1, 2) + view_10 = None + transpose_11 = key_layer_5.transpose(-1, -2) + key_layer_5 = None + attention_scores_6 = torch.matmul(query_layer_5, transpose_11) + query_layer_5 = transpose_11 = None + attention_scores_7 = attention_scores_6 / 8.0 + attention_scores_6 = None + attention_scores_8 = attention_scores_7 + extended_attention_mask_2 + attention_scores_7 = None + attention_probs_4 = torch.nn.functional.softmax(attention_scores_8, dim=-1) + attention_scores_8 = None + attention_probs_5 = torch.nn.functional.dropout( + attention_probs_4, 0.1, False, False + ) + attention_probs_4 = None + context_layer_6 = torch.matmul(attention_probs_5, value_layer_5) + attention_probs_5 = value_layer_5 = None + permute_2 = context_layer_6.permute(0, 2, 1, 3) + context_layer_6 = None + context_layer_7 = permute_2.contiguous() + permute_2 = None + context_layer_8 = context_layer_7.view((1, 11, 256)) + context_layer_7 = None + hidden_states_17 = torch._C._nn.linear( + context_layer_8, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_8 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_18 = torch.nn.functional.dropout( + hidden_states_17, 0.1, False, False + ) + hidden_states_17 = None + add_8 = hidden_states_18 + hidden_states_16 + hidden_states_18 = hidden_states_16 = None + hidden_states_19 = torch.nn.functional.layer_norm( + add_8, + (256,), + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_8 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_20 = torch._C._nn.linear( + hidden_states_19, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_21 = torch._C._nn.gelu(hidden_states_20) + hidden_states_20 = None + hidden_states_22 = torch._C._nn.linear( + hidden_states_21, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_21 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_23 = torch.nn.functional.dropout( + hidden_states_22, 0.1, False, False + ) + hidden_states_22 = None + add_9 = hidden_states_23 + hidden_states_19 + hidden_states_23 = hidden_states_19 = None + hidden_states_24 = torch.nn.functional.layer_norm( + add_9, + (256,), + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_9 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_6 = torch._C._nn.linear( + hidden_states_24, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_12 = query_layer_6.view(1, -1, 4, 64) + query_layer_6 = None + query_layer_7 = view_12.transpose(1, 2) + view_12 = None + key_layer_6 = torch._C._nn.linear( + hidden_states_24, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_13 = key_layer_6.view(1, -1, 4, 64) + key_layer_6 = None + key_layer_7 = view_13.transpose(1, 2) + view_13 = None + value_layer_6 = torch._C._nn.linear( + hidden_states_24, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_14 = value_layer_6.view(1, -1, 4, 64) + value_layer_6 = None + value_layer_7 = view_14.transpose(1, 2) + view_14 = None + transpose_15 = key_layer_7.transpose(-1, -2) + key_layer_7 = None + attention_scores_9 = torch.matmul(query_layer_7, transpose_15) + query_layer_7 = transpose_15 = None + attention_scores_10 = attention_scores_9 / 8.0 + attention_scores_9 = None + attention_scores_11 = attention_scores_10 + extended_attention_mask_2 + attention_scores_10 = None + attention_probs_6 = torch.nn.functional.softmax(attention_scores_11, dim=-1) + attention_scores_11 = None + attention_probs_7 = torch.nn.functional.dropout( + attention_probs_6, 0.1, False, False + ) + attention_probs_6 = None + context_layer_9 = torch.matmul(attention_probs_7, value_layer_7) + attention_probs_7 = value_layer_7 = None + permute_3 = context_layer_9.permute(0, 2, 1, 3) + context_layer_9 = None + context_layer_10 = permute_3.contiguous() + permute_3 = None + context_layer_11 = context_layer_10.view((1, 11, 256)) + context_layer_10 = None + hidden_states_25 = torch._C._nn.linear( + context_layer_11, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_11 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_26 = torch.nn.functional.dropout( + hidden_states_25, 0.1, False, False + ) + hidden_states_25 = None + add_11 = hidden_states_26 + hidden_states_24 + hidden_states_26 = hidden_states_24 = None + hidden_states_27 = torch.nn.functional.layer_norm( + add_11, + (256,), + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_11 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_28 = torch._C._nn.linear( + hidden_states_27, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_29 = torch._C._nn.gelu(hidden_states_28) + hidden_states_28 = None + hidden_states_30 = torch._C._nn.linear( + hidden_states_29, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_29 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_31 = torch.nn.functional.dropout( + hidden_states_30, 0.1, False, False + ) + hidden_states_30 = None + add_12 = hidden_states_31 + hidden_states_27 + hidden_states_31 = hidden_states_27 = None + hidden_states_32 = torch.nn.functional.layer_norm( + add_12, + (256,), + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_12 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_8 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_16 = query_layer_8.view(1, -1, 4, 64) + query_layer_8 = None + query_layer_9 = view_16.transpose(1, 2) + view_16 = None + key_layer_8 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_17 = key_layer_8.view(1, -1, 4, 64) + key_layer_8 = None + key_layer_9 = view_17.transpose(1, 2) + view_17 = None + value_layer_8 = torch._C._nn.linear( + hidden_states_32, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_18 = value_layer_8.view(1, -1, 4, 64) + value_layer_8 = None + value_layer_9 = view_18.transpose(1, 2) + view_18 = None + transpose_19 = key_layer_9.transpose(-1, -2) + key_layer_9 = None + attention_scores_12 = torch.matmul(query_layer_9, transpose_19) + query_layer_9 = transpose_19 = None + attention_scores_13 = attention_scores_12 / 8.0 + attention_scores_12 = None + attention_scores_14 = attention_scores_13 + extended_attention_mask_2 + attention_scores_13 = None + attention_probs_8 = torch.nn.functional.softmax(attention_scores_14, dim=-1) + attention_scores_14 = None + attention_probs_9 = torch.nn.functional.dropout( + attention_probs_8, 0.1, False, False + ) + attention_probs_8 = None + context_layer_12 = torch.matmul(attention_probs_9, value_layer_9) + attention_probs_9 = value_layer_9 = None + permute_4 = context_layer_12.permute(0, 2, 1, 3) + context_layer_12 = None + context_layer_13 = permute_4.contiguous() + permute_4 = None + context_layer_14 = context_layer_13.view((1, 11, 256)) + context_layer_13 = None + hidden_states_33 = torch._C._nn.linear( + context_layer_14, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_14 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_34 = torch.nn.functional.dropout( + hidden_states_33, 0.1, False, False + ) + hidden_states_33 = None + add_14 = hidden_states_34 + hidden_states_32 + hidden_states_34 = hidden_states_32 = None + hidden_states_35 = torch.nn.functional.layer_norm( + add_14, + (256,), + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_14 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_36 = torch._C._nn.linear( + hidden_states_35, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_37 = torch._C._nn.gelu(hidden_states_36) + hidden_states_36 = None + hidden_states_38 = torch._C._nn.linear( + hidden_states_37, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_37 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_39 = torch.nn.functional.dropout( + hidden_states_38, 0.1, False, False + ) + hidden_states_38 = None + add_15 = hidden_states_39 + hidden_states_35 + hidden_states_39 = hidden_states_35 = None + hidden_states_40 = torch.nn.functional.layer_norm( + add_15, + (256,), + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_15 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_10 = torch._C._nn.linear( + hidden_states_40, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_20 = query_layer_10.view(1, -1, 4, 64) + query_layer_10 = None + query_layer_11 = view_20.transpose(1, 2) + view_20 = None + key_layer_10 = torch._C._nn.linear( + hidden_states_40, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_21 = key_layer_10.view(1, -1, 4, 64) + key_layer_10 = None + key_layer_11 = view_21.transpose(1, 2) + view_21 = None + value_layer_10 = torch._C._nn.linear( + hidden_states_40, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_22 = value_layer_10.view(1, -1, 4, 64) + value_layer_10 = None + value_layer_11 = view_22.transpose(1, 2) + view_22 = None + transpose_23 = key_layer_11.transpose(-1, -2) + key_layer_11 = None + attention_scores_15 = torch.matmul(query_layer_11, transpose_23) + query_layer_11 = transpose_23 = None + attention_scores_16 = attention_scores_15 / 8.0 + attention_scores_15 = None + attention_scores_17 = attention_scores_16 + extended_attention_mask_2 + attention_scores_16 = None + attention_probs_10 = torch.nn.functional.softmax(attention_scores_17, dim=-1) + attention_scores_17 = None + attention_probs_11 = torch.nn.functional.dropout( + attention_probs_10, 0.1, False, False + ) + attention_probs_10 = None + context_layer_15 = torch.matmul(attention_probs_11, value_layer_11) + attention_probs_11 = value_layer_11 = None + permute_5 = context_layer_15.permute(0, 2, 1, 3) + context_layer_15 = None + context_layer_16 = permute_5.contiguous() + permute_5 = None + context_layer_17 = context_layer_16.view((1, 11, 256)) + context_layer_16 = None + hidden_states_41 = torch._C._nn.linear( + context_layer_17, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_17 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_42 = torch.nn.functional.dropout( + hidden_states_41, 0.1, False, False + ) + hidden_states_41 = None + add_17 = hidden_states_42 + hidden_states_40 + hidden_states_42 = hidden_states_40 = None + hidden_states_43 = torch.nn.functional.layer_norm( + add_17, + (256,), + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_17 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_44 = torch._C._nn.linear( + hidden_states_43, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_45 = torch._C._nn.gelu(hidden_states_44) + hidden_states_44 = None + hidden_states_46 = torch._C._nn.linear( + hidden_states_45, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_45 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_47 = torch.nn.functional.dropout( + hidden_states_46, 0.1, False, False + ) + hidden_states_46 = None + add_18 = hidden_states_47 + hidden_states_43 + hidden_states_47 = hidden_states_43 = None + hidden_states_48 = torch.nn.functional.layer_norm( + add_18, + (256,), + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_18 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_12 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_24 = query_layer_12.view(1, -1, 4, 64) + query_layer_12 = None + query_layer_13 = view_24.transpose(1, 2) + view_24 = None + key_layer_12 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_25 = key_layer_12.view(1, -1, 4, 64) + key_layer_12 = None + key_layer_13 = view_25.transpose(1, 2) + view_25 = None + value_layer_12 = torch._C._nn.linear( + hidden_states_48, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_26 = value_layer_12.view(1, -1, 4, 64) + value_layer_12 = None + value_layer_13 = view_26.transpose(1, 2) + view_26 = None + transpose_27 = key_layer_13.transpose(-1, -2) + key_layer_13 = None + attention_scores_18 = torch.matmul(query_layer_13, transpose_27) + query_layer_13 = transpose_27 = None + attention_scores_19 = attention_scores_18 / 8.0 + attention_scores_18 = None + attention_scores_20 = attention_scores_19 + extended_attention_mask_2 + attention_scores_19 = None + attention_probs_12 = torch.nn.functional.softmax(attention_scores_20, dim=-1) + attention_scores_20 = None + attention_probs_13 = torch.nn.functional.dropout( + attention_probs_12, 0.1, False, False + ) + attention_probs_12 = None + context_layer_18 = torch.matmul(attention_probs_13, value_layer_13) + attention_probs_13 = value_layer_13 = None + permute_6 = context_layer_18.permute(0, 2, 1, 3) + context_layer_18 = None + context_layer_19 = permute_6.contiguous() + permute_6 = None + context_layer_20 = context_layer_19.view((1, 11, 256)) + context_layer_19 = None + hidden_states_49 = torch._C._nn.linear( + context_layer_20, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_20 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_50 = torch.nn.functional.dropout( + hidden_states_49, 0.1, False, False + ) + hidden_states_49 = None + add_20 = hidden_states_50 + hidden_states_48 + hidden_states_50 = hidden_states_48 = None + hidden_states_51 = torch.nn.functional.layer_norm( + add_20, + (256,), + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_20 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_52 = torch._C._nn.linear( + hidden_states_51, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_53 = torch._C._nn.gelu(hidden_states_52) + hidden_states_52 = None + hidden_states_54 = torch._C._nn.linear( + hidden_states_53, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_53 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_55 = torch.nn.functional.dropout( + hidden_states_54, 0.1, False, False + ) + hidden_states_54 = None + add_21 = hidden_states_55 + hidden_states_51 + hidden_states_55 = hidden_states_51 = None + hidden_states_56 = torch.nn.functional.layer_norm( + add_21, + (256,), + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_21 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_14 = torch._C._nn.linear( + hidden_states_56, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_28 = query_layer_14.view(1, -1, 4, 64) + query_layer_14 = None + query_layer_15 = view_28.transpose(1, 2) + view_28 = None + key_layer_14 = torch._C._nn.linear( + hidden_states_56, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_29 = key_layer_14.view(1, -1, 4, 64) + key_layer_14 = None + key_layer_15 = view_29.transpose(1, 2) + view_29 = None + value_layer_14 = torch._C._nn.linear( + hidden_states_56, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_30 = value_layer_14.view(1, -1, 4, 64) + value_layer_14 = None + value_layer_15 = view_30.transpose(1, 2) + view_30 = None + transpose_31 = key_layer_15.transpose(-1, -2) + key_layer_15 = None + attention_scores_21 = torch.matmul(query_layer_15, transpose_31) + query_layer_15 = transpose_31 = None + attention_scores_22 = attention_scores_21 / 8.0 + attention_scores_21 = None + attention_scores_23 = attention_scores_22 + extended_attention_mask_2 + attention_scores_22 = None + attention_probs_14 = torch.nn.functional.softmax(attention_scores_23, dim=-1) + attention_scores_23 = None + attention_probs_15 = torch.nn.functional.dropout( + attention_probs_14, 0.1, False, False + ) + attention_probs_14 = None + context_layer_21 = torch.matmul(attention_probs_15, value_layer_15) + attention_probs_15 = value_layer_15 = None + permute_7 = context_layer_21.permute(0, 2, 1, 3) + context_layer_21 = None + context_layer_22 = permute_7.contiguous() + permute_7 = None + context_layer_23 = context_layer_22.view((1, 11, 256)) + context_layer_22 = None + hidden_states_57 = torch._C._nn.linear( + context_layer_23, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_23 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_58 = torch.nn.functional.dropout( + hidden_states_57, 0.1, False, False + ) + hidden_states_57 = None + add_23 = hidden_states_58 + hidden_states_56 + hidden_states_58 = hidden_states_56 = None + hidden_states_59 = torch.nn.functional.layer_norm( + add_23, + (256,), + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_23 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_60 = torch._C._nn.linear( + hidden_states_59, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_61 = torch._C._nn.gelu(hidden_states_60) + hidden_states_60 = None + hidden_states_62 = torch._C._nn.linear( + hidden_states_61, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_61 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_63 = torch.nn.functional.dropout( + hidden_states_62, 0.1, False, False + ) + hidden_states_62 = None + add_24 = hidden_states_63 + hidden_states_59 + hidden_states_63 = hidden_states_59 = None + hidden_states_64 = torch.nn.functional.layer_norm( + add_24, + (256,), + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_24 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_16 = torch._C._nn.linear( + hidden_states_64, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_32 = query_layer_16.view(1, -1, 4, 64) + query_layer_16 = None + query_layer_17 = view_32.transpose(1, 2) + view_32 = None + key_layer_16 = torch._C._nn.linear( + hidden_states_64, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_33 = key_layer_16.view(1, -1, 4, 64) + key_layer_16 = None + key_layer_17 = view_33.transpose(1, 2) + view_33 = None + value_layer_16 = torch._C._nn.linear( + hidden_states_64, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_34 = value_layer_16.view(1, -1, 4, 64) + value_layer_16 = None + value_layer_17 = view_34.transpose(1, 2) + view_34 = None + transpose_35 = key_layer_17.transpose(-1, -2) + key_layer_17 = None + attention_scores_24 = torch.matmul(query_layer_17, transpose_35) + query_layer_17 = transpose_35 = None + attention_scores_25 = attention_scores_24 / 8.0 + attention_scores_24 = None + attention_scores_26 = attention_scores_25 + extended_attention_mask_2 + attention_scores_25 = None + attention_probs_16 = torch.nn.functional.softmax(attention_scores_26, dim=-1) + attention_scores_26 = None + attention_probs_17 = torch.nn.functional.dropout( + attention_probs_16, 0.1, False, False + ) + attention_probs_16 = None + context_layer_24 = torch.matmul(attention_probs_17, value_layer_17) + attention_probs_17 = value_layer_17 = None + permute_8 = context_layer_24.permute(0, 2, 1, 3) + context_layer_24 = None + context_layer_25 = permute_8.contiguous() + permute_8 = None + context_layer_26 = context_layer_25.view((1, 11, 256)) + context_layer_25 = None + hidden_states_65 = torch._C._nn.linear( + context_layer_26, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_26 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_66 = torch.nn.functional.dropout( + hidden_states_65, 0.1, False, False + ) + hidden_states_65 = None + add_26 = hidden_states_66 + hidden_states_64 + hidden_states_66 = hidden_states_64 = None + hidden_states_67 = torch.nn.functional.layer_norm( + add_26, + (256,), + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_26 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_68 = torch._C._nn.linear( + hidden_states_67, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_69 = torch._C._nn.gelu(hidden_states_68) + hidden_states_68 = None + hidden_states_70 = torch._C._nn.linear( + hidden_states_69, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_69 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_71 = torch.nn.functional.dropout( + hidden_states_70, 0.1, False, False + ) + hidden_states_70 = None + add_27 = hidden_states_71 + hidden_states_67 + hidden_states_71 = hidden_states_67 = None + hidden_states_72 = torch.nn.functional.layer_norm( + add_27, + (256,), + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_27 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_18 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_36 = query_layer_18.view(1, -1, 4, 64) + query_layer_18 = None + query_layer_19 = view_36.transpose(1, 2) + view_36 = None + key_layer_18 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_37 = key_layer_18.view(1, -1, 4, 64) + key_layer_18 = None + key_layer_19 = view_37.transpose(1, 2) + view_37 = None + value_layer_18 = torch._C._nn.linear( + hidden_states_72, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_38 = value_layer_18.view(1, -1, 4, 64) + value_layer_18 = None + value_layer_19 = view_38.transpose(1, 2) + view_38 = None + transpose_39 = key_layer_19.transpose(-1, -2) + key_layer_19 = None + attention_scores_27 = torch.matmul(query_layer_19, transpose_39) + query_layer_19 = transpose_39 = None + attention_scores_28 = attention_scores_27 / 8.0 + attention_scores_27 = None + attention_scores_29 = attention_scores_28 + extended_attention_mask_2 + attention_scores_28 = None + attention_probs_18 = torch.nn.functional.softmax(attention_scores_29, dim=-1) + attention_scores_29 = None + attention_probs_19 = torch.nn.functional.dropout( + attention_probs_18, 0.1, False, False + ) + attention_probs_18 = None + context_layer_27 = torch.matmul(attention_probs_19, value_layer_19) + attention_probs_19 = value_layer_19 = None + permute_9 = context_layer_27.permute(0, 2, 1, 3) + context_layer_27 = None + context_layer_28 = permute_9.contiguous() + permute_9 = None + context_layer_29 = context_layer_28.view((1, 11, 256)) + context_layer_28 = None + hidden_states_73 = torch._C._nn.linear( + context_layer_29, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_29 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_74 = torch.nn.functional.dropout( + hidden_states_73, 0.1, False, False + ) + hidden_states_73 = None + add_29 = hidden_states_74 + hidden_states_72 + hidden_states_74 = hidden_states_72 = None + hidden_states_75 = torch.nn.functional.layer_norm( + add_29, + (256,), + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_29 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_76 = torch._C._nn.linear( + hidden_states_75, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_77 = torch._C._nn.gelu(hidden_states_76) + hidden_states_76 = None + hidden_states_78 = torch._C._nn.linear( + hidden_states_77, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_77 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_79 = torch.nn.functional.dropout( + hidden_states_78, 0.1, False, False + ) + hidden_states_78 = None + add_30 = hidden_states_79 + hidden_states_75 + hidden_states_79 = hidden_states_75 = None + hidden_states_80 = torch.nn.functional.layer_norm( + add_30, + (256,), + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_30 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_20 = torch._C._nn.linear( + hidden_states_80, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_40 = query_layer_20.view(1, -1, 4, 64) + query_layer_20 = None + query_layer_21 = view_40.transpose(1, 2) + view_40 = None + key_layer_20 = torch._C._nn.linear( + hidden_states_80, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_41 = key_layer_20.view(1, -1, 4, 64) + key_layer_20 = None + key_layer_21 = view_41.transpose(1, 2) + view_41 = None + value_layer_20 = torch._C._nn.linear( + hidden_states_80, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_42 = value_layer_20.view(1, -1, 4, 64) + value_layer_20 = None + value_layer_21 = view_42.transpose(1, 2) + view_42 = None + transpose_43 = key_layer_21.transpose(-1, -2) + key_layer_21 = None + attention_scores_30 = torch.matmul(query_layer_21, transpose_43) + query_layer_21 = transpose_43 = None + attention_scores_31 = attention_scores_30 / 8.0 + attention_scores_30 = None + attention_scores_32 = attention_scores_31 + extended_attention_mask_2 + attention_scores_31 = None + attention_probs_20 = torch.nn.functional.softmax(attention_scores_32, dim=-1) + attention_scores_32 = None + attention_probs_21 = torch.nn.functional.dropout( + attention_probs_20, 0.1, False, False + ) + attention_probs_20 = None + context_layer_30 = torch.matmul(attention_probs_21, value_layer_21) + attention_probs_21 = value_layer_21 = None + permute_10 = context_layer_30.permute(0, 2, 1, 3) + context_layer_30 = None + context_layer_31 = permute_10.contiguous() + permute_10 = None + context_layer_32 = context_layer_31.view((1, 11, 256)) + context_layer_31 = None + hidden_states_81 = torch._C._nn.linear( + context_layer_32, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_32 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_82 = torch.nn.functional.dropout( + hidden_states_81, 0.1, False, False + ) + hidden_states_81 = None + add_32 = hidden_states_82 + hidden_states_80 + hidden_states_82 = hidden_states_80 = None + hidden_states_83 = torch.nn.functional.layer_norm( + add_32, + (256,), + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_32 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_84 = torch._C._nn.linear( + hidden_states_83, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_85 = torch._C._nn.gelu(hidden_states_84) + hidden_states_84 = None + hidden_states_86 = torch._C._nn.linear( + hidden_states_85, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_85 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_87 = torch.nn.functional.dropout( + hidden_states_86, 0.1, False, False + ) + hidden_states_86 = None + add_33 = hidden_states_87 + hidden_states_83 + hidden_states_87 = hidden_states_83 = None + hidden_states_88 = torch.nn.functional.layer_norm( + add_33, + (256,), + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_33 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = (None) + query_layer_22 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_44 = query_layer_22.view(1, -1, 4, 64) + query_layer_22 = None + query_layer_23 = view_44.transpose(1, 2) + view_44 = None + key_layer_22 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_45 = key_layer_22.view(1, -1, 4, 64) + key_layer_22 = None + key_layer_23 = view_45.transpose(1, 2) + view_45 = None + value_layer_22 = torch._C._nn.linear( + hidden_states_88, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_46 = value_layer_22.view(1, -1, 4, 64) + value_layer_22 = None + value_layer_23 = view_46.transpose(1, 2) + view_46 = None + transpose_47 = key_layer_23.transpose(-1, -2) + key_layer_23 = None + attention_scores_33 = torch.matmul(query_layer_23, transpose_47) + query_layer_23 = transpose_47 = None + attention_scores_34 = attention_scores_33 / 8.0 + attention_scores_33 = None + attention_scores_35 = attention_scores_34 + extended_attention_mask_2 + attention_scores_34 = extended_attention_mask_2 = None + attention_probs_22 = torch.nn.functional.softmax(attention_scores_35, dim=-1) + attention_scores_35 = None + attention_probs_23 = torch.nn.functional.dropout( + attention_probs_22, 0.1, False, False + ) + attention_probs_22 = None + context_layer_33 = torch.matmul(attention_probs_23, value_layer_23) + attention_probs_23 = value_layer_23 = None + permute_11 = context_layer_33.permute(0, 2, 1, 3) + context_layer_33 = None + context_layer_34 = permute_11.contiguous() + permute_11 = None + context_layer_35 = context_layer_34.view((1, 11, 256)) + context_layer_34 = None + hidden_states_89 = torch._C._nn.linear( + context_layer_35, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + context_layer_35 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_90 = torch.nn.functional.dropout( + hidden_states_89, 0.1, False, False + ) + hidden_states_89 = None + add_35 = hidden_states_90 + hidden_states_88 + hidden_states_90 = hidden_states_88 = None + hidden_states_91 = torch.nn.functional.layer_norm( + add_35, + (256,), + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_35 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_92 = torch._C._nn.linear( + hidden_states_91, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_93 = torch._C._nn.gelu(hidden_states_92) + hidden_states_92 = None + hidden_states_94 = torch._C._nn.linear( + hidden_states_93, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_93 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_95 = torch.nn.functional.dropout( + hidden_states_94, 0.1, False, False + ) + hidden_states_94 = None + add_36 = hidden_states_95 + hidden_states_91 + hidden_states_95 = hidden_states_91 = None + hidden_states_96 = torch.nn.functional.layer_norm( + add_36, + (256,), + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_36 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = (None) + return (hidden_states_96,) diff --git a/samples/transformers-auto-model/google_electra-small-discriminator/weight_meta.py b/samples/transformers-auto-model/google_electra-small-discriminator/weight_meta.py new file mode 100644 index 0000000000..8575082642 --- /dev/null +++ b/samples/transformers-auto-model/google_electra-small-discriminator/weight_meta.py @@ -0,0 +1,2541 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [101, 10629, 7159, 2003, 2109, 2000, 14817, 15078, 19287, 1012, 102] + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_token_type_ids_: + name = "L_token_type_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_word_embeddings_parameters_weight_" + shape = [30522, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_buffers_position_ids_: + name = "L_self_modules_embeddings_buffers_position_ids_" + shape = [1, 512] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356, + 357, + 358, + 359, + 360, + 361, + 362, + 363, + 364, + 365, + 366, + 367, + 368, + 369, + 370, + 371, + 372, + 373, + 374, + 375, + 376, + 377, + 378, + 379, + 380, + 381, + 382, + 383, + 384, + 385, + 386, + 387, + 388, + 389, + 390, + 391, + 392, + 393, + 394, + 395, + 396, + 397, + 398, + 399, + 400, + 401, + 402, + 403, + 404, + 405, + 406, + 407, + 408, + 409, + 410, + 411, + 412, + 413, + 414, + 415, + 416, + 417, + 418, + 419, + 420, + 421, + 422, + 423, + 424, + 425, + 426, + 427, + 428, + 429, + 430, + 431, + 432, + 433, + 434, + 435, + 436, + 437, + 438, + 439, + 440, + 441, + 442, + 443, + 444, + 445, + 446, + 447, + 448, + 449, + 450, + 451, + 452, + 453, + 454, + 455, + 456, + 457, + 458, + 459, + 460, + 461, + 462, + 463, + 464, + 465, + 466, + 467, + 468, + 469, + 470, + 471, + 472, + 473, + 474, + 475, + 476, + 477, + 478, + 479, + 480, + 481, + 482, + 483, + 484, + 485, + 486, + 487, + 488, + 489, + 490, + 491, + 492, + 493, + 494, + 495, + 496, + 497, + 498, + 499, + 500, + 501, + 502, + 503, + 504, + 505, + 506, + 507, + 508, + 509, + 510, + 511, + ] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_" + shape = [2, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.024 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_position_embeddings_parameters_weight_" + shape = [512, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_weight_" + shape = [128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.885 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_bias_" + shape = [128] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.377 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_project_parameters_weight_: + name = "L_self_modules_embeddings_project_parameters_weight_" + shape = [256, 128] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_project_parameters_bias_: + name = "L_self_modules_embeddings_project_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.151 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.016 + std = 0.207 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.104 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.913 + std = 0.175 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.038 + std = 0.426 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.485 + std = 0.279 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.200 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.242 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.053 + std = 0.273 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.702 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.160 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.242 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.950 + std = 0.122 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.295 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.497 + std = 0.347 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.126 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.019 + std = 0.194 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.044 + std = 0.598 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.129 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.187 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.959 + std = 0.120 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.080 + std = 0.263 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.475 + std = 0.301 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.101 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.224 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.154 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.033 + std = 0.558 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.157 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.218 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.938 + std = 0.133 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.017 + std = 0.223 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.483 + std = 0.303 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.506 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.038 + std = 0.182 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.031 + std = 0.527 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.133 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.217 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.943 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.223 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.473 + std = 0.285 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.262 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.026 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.570 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.159 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.989 + std = 0.147 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.043 + std = 0.184 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.483 + std = 0.262 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.261 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.021 + std = 0.119 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.047 + std = 0.533 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.148 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.159 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.945 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.128 + std = 0.179 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.497 + std = 0.262 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.170 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.173 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.046 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.033 + std = 0.671 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.151 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.118 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.923 + std = 0.154 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.140 + std = 0.150 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.474 + std = 0.260 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.185 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.240 + std = 0.095 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.026 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.068 + std = 0.558 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.118 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.159 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.813 + std = 0.173 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.078 + std = 0.171 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.473 + std = 0.258 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.185 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.452 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.076 + std = 0.185 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.046 + std = 0.521 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.014 + std = 0.162 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.163 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.330 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.779 + std = 0.149 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.034 + std = 0.191 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.415 + std = 0.248 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.370 + std = 0.113 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.023 + std = 0.164 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.030 + std = 0.411 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.144 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.147 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.214 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.773 + std = 0.199 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.093 + std = 0.221 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.435 + std = 0.230 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.184 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.148 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.030 + std = 0.162 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.058 + std = 0.403 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.009 + std = 0.154 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.078 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.023 + std = 0.246 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [256, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.317 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.663 + std = 0.171 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.091 + std = 0.207 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_" + shape = [1024, 256] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_" + shape = [1024] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.385 + std = 0.177 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_" + shape = [256, 1024] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.161 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.812 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_" + shape = [256] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.070 + std = 0.082 + data = None diff --git a/samples/transformers-auto-model/gpt2/graph_hash.txt b/samples/transformers-auto-model/gpt2/graph_hash.txt new file mode 100644 index 0000000000..655be80158 --- /dev/null +++ b/samples/transformers-auto-model/gpt2/graph_hash.txt @@ -0,0 +1 @@ +a9e02d26078bcd9f86a6795c666e1bbc762855650d62fd74760444ddeaf01d62 \ No newline at end of file diff --git a/samples/transformers-auto-model/gpt2/graph_net.json b/samples/transformers-auto-model/gpt2/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/gpt2/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/gpt2/input_meta.py b/samples/transformers-auto-model/gpt2/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/gpt2/input_tensor_constraints.py b/samples/transformers-auto-model/gpt2/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/gpt2/model.py b/samples/transformers-auto-model/gpt2/model.py new file mode 100644 index 0000000000..c75323a546 --- /dev/null +++ b/samples/transformers-auto-model/gpt2/model.py @@ -0,0 +1,2490 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_self_modules_wte_parameters_weight_: torch.nn.parameter.Parameter, + L_input_ids_: torch.Tensor, + L_self_modules_wpe_parameters_weight_: torch.nn.parameter.Parameter, + L_attention_mask_: torch.Tensor, + L_self_modules_h_modules_0_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_ln_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_ln_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_ln_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_ln_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_ln_f_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_ln_f_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_self_modules_wte_parameters_weight_ = L_self_modules_wte_parameters_weight_ + l_input_ids_ = L_input_ids_ + l_self_modules_wpe_parameters_weight_ = L_self_modules_wpe_parameters_weight_ + l_attention_mask_ = L_attention_mask_ + l_self_modules_h_modules_0_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_0_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_0_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_0_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_ln_1_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_ln_1_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_ln_1_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_ln_1_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_ln_2_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_ln_2_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_ln_2_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_ln_2_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ = ( + L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ = ( + L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ + ) + l_self_modules_ln_f_parameters_weight_ = L_self_modules_ln_f_parameters_weight_ + l_self_modules_ln_f_parameters_bias_ = L_self_modules_ln_f_parameters_bias_ + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_wte_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + l_input_ids_ = l_self_modules_wte_parameters_weight_ = None + cache_position = torch.arange(0, 9, device=device(type="cuda", index=0)) + position_ids = cache_position.unsqueeze(0) + position_embeds = torch.nn.functional.embedding( + position_ids, + l_self_modules_wpe_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + position_ids = l_self_modules_wpe_parameters_weight_ = None + to = position_embeds.to(device(type="cuda", index=0)) + position_embeds = None + hidden_states = inputs_embeds + to + inputs_embeds = to = None + attention_mask = l_attention_mask_.view(1, -1) + l_attention_mask_ = None + attention_mask_1 = attention_mask.to( + device=device(type="cuda", index=0), dtype=torch.bool + ) + attention_mask = None + kv_arange = torch.arange(9, device=device(type="cuda", index=0)) + kv_arange += 0 + kv_arange_1 = kv_arange + kv_arange = None + batch_arange = torch.arange(1, device=device(type="cuda", index=0)) + head_arange = torch.arange(1, device=device(type="cuda", index=0)) + lazy_load_decompositions = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions = None + _vmap_increment_nesting = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting = None + child = torch._C._functorch._add_batch_dim(batch_arange, 0, 1) + batch_arange = None + lazy_load_decompositions_1 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_1 = None + _vmap_increment_nesting_1 = torch._C._functorch._vmap_increment_nesting( + 1, "error" + ) + _vmap_increment_nesting_1 = None + child_1 = torch._C._functorch._add_batch_dim(head_arange, 0, 2) + head_arange = child_1 = None + lazy_load_decompositions_2 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_2 = None + _vmap_increment_nesting_2 = torch._C._functorch._vmap_increment_nesting( + 9, "error" + ) + _vmap_increment_nesting_2 = None + child_2 = torch._C._functorch._add_batch_dim(cache_position, 0, 3) + cache_position = None + lazy_load_decompositions_3 = torch._functorch.vmap.lazy_load_decompositions() + lazy_load_decompositions_3 = None + _vmap_increment_nesting_3 = torch._C._functorch._vmap_increment_nesting( + 9, "error" + ) + _vmap_increment_nesting_3 = None + child_3 = torch._C._functorch._add_batch_dim(kv_arange_1, 0, 4) + kv_arange_1 = None + result = child_2.new_ones((), dtype=torch.bool) + le = child_3.le(child_2) + child_2 = None + result_1 = result.__and__(le) + result = le = None + index = torch.ops.aten.index(attention_mask_1, [child, child_3]) + attention_mask_1 = child = child_3 = None + result_2 = result_1.__and__(index) + result_1 = index = None + batched_outputs = torch._C._functorch._remove_batch_dim(result_2, 4, 9, 0) + result_2 = None + _vmap_decrement_nesting = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting = None + batched_outputs_1 = torch._C._functorch._remove_batch_dim( + batched_outputs, 3, 9, 0 + ) + batched_outputs = None + _vmap_decrement_nesting_1 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_1 = None + batched_outputs_2 = torch._C._functorch._remove_batch_dim( + batched_outputs_1, 2, 1, 0 + ) + batched_outputs_1 = None + _vmap_decrement_nesting_2 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_2 = None + causal_mask = torch._C._functorch._remove_batch_dim(batched_outputs_2, 1, 1, 0) + batched_outputs_2 = None + _vmap_decrement_nesting_3 = torch._C._functorch._vmap_decrement_nesting() + _vmap_decrement_nesting_3 = None + hidden_states_1 = torch.nn.functional.dropout(hidden_states, 0.1, False, False) + hidden_states = None + hidden_states_2 = torch.nn.functional.layer_norm( + hidden_states_1, + (768,), + l_self_modules_h_modules_0_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_0_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_0_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_0_modules_ln_1_parameters_bias_ + ) = None + view_1 = hidden_states_2.view(-1, 768) + hidden_states_2 = None + x = torch.addmm( + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_, + view_1, + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_ = ( + view_1 + ) = ( + l_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_1 = x.view((1, 9, 2304)) + x = None + split = x_1.split(768, dim=2) + x_1 = None + query_states = split[0] + key_states = split[1] + value_states = split[2] + split = None + view_3 = key_states.view((1, 9, -1, 64)) + key_states = None + key_states_1 = view_3.transpose(1, 2) + view_3 = None + view_4 = value_states.view((1, 9, -1, 64)) + value_states = None + value_states_1 = view_4.transpose(1, 2) + view_4 = None + view_5 = query_states.view((1, 9, -1, 64)) + query_states = None + query_states_1 = view_5.transpose(1, 2) + view_5 = None + attention_mask_2 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query = query_states_1.contiguous() + query_states_1 = None + key = key_states_1.contiguous() + value = value_states_1.contiguous() + attn_output = torch._C._nn.scaled_dot_product_attention( + query, + key, + value, + attn_mask=attention_mask_2, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query = key = value = attention_mask_2 = None + transpose_3 = attn_output.transpose(1, 2) + attn_output = None + attn_output_1 = transpose_3.contiguous() + transpose_3 = None + reshape = attn_output_1.reshape(1, 9, -1) + attn_output_1 = None + attn_output_2 = reshape.contiguous() + reshape = None + view_6 = attn_output_2.view(-1, 768) + attn_output_2 = None + x_2 = torch.addmm( + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_, + view_6, + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_ = ( + view_6 + ) = ( + l_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_3 = x_2.view((1, 9, 768)) + x_2 = None + attn_output_3 = torch.nn.functional.dropout(x_3, 0.1, False, False) + x_3 = None + hidden_states_3 = attn_output_3 + hidden_states_1 + attn_output_3 = hidden_states_1 = None + hidden_states_4 = torch.nn.functional.layer_norm( + hidden_states_3, + (768,), + l_self_modules_h_modules_0_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_0_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_0_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_0_modules_ln_2_parameters_bias_ + ) = None + view_8 = hidden_states_4.view(-1, 768) + hidden_states_4 = None + x_4 = torch.addmm( + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_, + view_8, + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_8 + ) = ( + l_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_5 = x_4.view((1, 9, 3072)) + x_4 = None + mul = 0.5 * x_5 + pow_1 = torch.pow(x_5, 3.0) + mul_1 = 0.044715 * pow_1 + pow_1 = None + add_2 = x_5 + mul_1 + x_5 = mul_1 = None + mul_2 = 0.7978845608028654 * add_2 + add_2 = None + tanh = torch.tanh(mul_2) + mul_2 = None + add_3 = 1.0 + tanh + tanh = None + hidden_states_5 = mul * add_3 + mul = add_3 = None + view_10 = hidden_states_5.view(-1, 3072) + hidden_states_5 = None + x_6 = torch.addmm( + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_, + view_10, + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_10 + ) = ( + l_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_7 = x_6.view((1, 9, 768)) + x_6 = None + hidden_states_6 = torch.nn.functional.dropout(x_7, 0.1, False, False) + x_7 = None + hidden_states_7 = hidden_states_3 + hidden_states_6 + hidden_states_3 = hidden_states_6 = None + hidden_states_8 = torch.nn.functional.layer_norm( + hidden_states_7, + (768,), + l_self_modules_h_modules_1_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_1_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_1_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_1_modules_ln_1_parameters_bias_ + ) = None + view_12 = hidden_states_8.view(-1, 768) + hidden_states_8 = None + x_8 = torch.addmm( + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_, + view_12, + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_ = ( + view_12 + ) = ( + l_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_9 = x_8.view((1, 9, 2304)) + x_8 = None + split_1 = x_9.split(768, dim=2) + x_9 = None + query_states_2 = split_1[0] + key_states_2 = split_1[1] + value_states_2 = split_1[2] + split_1 = None + view_14 = key_states_2.view((1, 9, -1, 64)) + key_states_2 = None + key_states_3 = view_14.transpose(1, 2) + view_14 = None + view_15 = value_states_2.view((1, 9, -1, 64)) + value_states_2 = None + value_states_3 = view_15.transpose(1, 2) + view_15 = None + view_16 = query_states_2.view((1, 9, -1, 64)) + query_states_2 = None + query_states_3 = view_16.transpose(1, 2) + view_16 = None + attention_mask_3 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_1 = query_states_3.contiguous() + query_states_3 = None + key_1 = key_states_3.contiguous() + value_1 = value_states_3.contiguous() + attn_output_4 = torch._C._nn.scaled_dot_product_attention( + query_1, + key_1, + value_1, + attn_mask=attention_mask_3, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_1 = key_1 = value_1 = attention_mask_3 = None + transpose_7 = attn_output_4.transpose(1, 2) + attn_output_4 = None + attn_output_5 = transpose_7.contiguous() + transpose_7 = None + reshape_1 = attn_output_5.reshape(1, 9, -1) + attn_output_5 = None + attn_output_6 = reshape_1.contiguous() + reshape_1 = None + view_17 = attn_output_6.view(-1, 768) + attn_output_6 = None + x_10 = torch.addmm( + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_, + view_17, + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_ = ( + view_17 + ) = ( + l_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_11 = x_10.view((1, 9, 768)) + x_10 = None + attn_output_7 = torch.nn.functional.dropout(x_11, 0.1, False, False) + x_11 = None + hidden_states_9 = attn_output_7 + hidden_states_7 + attn_output_7 = hidden_states_7 = None + hidden_states_10 = torch.nn.functional.layer_norm( + hidden_states_9, + (768,), + l_self_modules_h_modules_1_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_1_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_1_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_1_modules_ln_2_parameters_bias_ + ) = None + view_19 = hidden_states_10.view(-1, 768) + hidden_states_10 = None + x_12 = torch.addmm( + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_, + view_19, + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_19 + ) = ( + l_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_13 = x_12.view((1, 9, 3072)) + x_12 = None + mul_4 = 0.5 * x_13 + pow_2 = torch.pow(x_13, 3.0) + mul_5 = 0.044715 * pow_2 + pow_2 = None + add_6 = x_13 + mul_5 + x_13 = mul_5 = None + mul_6 = 0.7978845608028654 * add_6 + add_6 = None + tanh_1 = torch.tanh(mul_6) + mul_6 = None + add_7 = 1.0 + tanh_1 + tanh_1 = None + hidden_states_11 = mul_4 * add_7 + mul_4 = add_7 = None + view_21 = hidden_states_11.view(-1, 3072) + hidden_states_11 = None + x_14 = torch.addmm( + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_, + view_21, + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_21 + ) = ( + l_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_15 = x_14.view((1, 9, 768)) + x_14 = None + hidden_states_12 = torch.nn.functional.dropout(x_15, 0.1, False, False) + x_15 = None + hidden_states_13 = hidden_states_9 + hidden_states_12 + hidden_states_9 = hidden_states_12 = None + hidden_states_14 = torch.nn.functional.layer_norm( + hidden_states_13, + (768,), + l_self_modules_h_modules_2_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_2_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_2_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_2_modules_ln_1_parameters_bias_ + ) = None + view_23 = hidden_states_14.view(-1, 768) + hidden_states_14 = None + x_16 = torch.addmm( + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_, + view_23, + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_ = ( + view_23 + ) = ( + l_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_17 = x_16.view((1, 9, 2304)) + x_16 = None + split_2 = x_17.split(768, dim=2) + x_17 = None + query_states_4 = split_2[0] + key_states_4 = split_2[1] + value_states_4 = split_2[2] + split_2 = None + view_25 = key_states_4.view((1, 9, -1, 64)) + key_states_4 = None + key_states_5 = view_25.transpose(1, 2) + view_25 = None + view_26 = value_states_4.view((1, 9, -1, 64)) + value_states_4 = None + value_states_5 = view_26.transpose(1, 2) + view_26 = None + view_27 = query_states_4.view((1, 9, -1, 64)) + query_states_4 = None + query_states_5 = view_27.transpose(1, 2) + view_27 = None + attention_mask_4 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_2 = query_states_5.contiguous() + query_states_5 = None + key_2 = key_states_5.contiguous() + value_2 = value_states_5.contiguous() + attn_output_8 = torch._C._nn.scaled_dot_product_attention( + query_2, + key_2, + value_2, + attn_mask=attention_mask_4, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_2 = key_2 = value_2 = attention_mask_4 = None + transpose_11 = attn_output_8.transpose(1, 2) + attn_output_8 = None + attn_output_9 = transpose_11.contiguous() + transpose_11 = None + reshape_2 = attn_output_9.reshape(1, 9, -1) + attn_output_9 = None + attn_output_10 = reshape_2.contiguous() + reshape_2 = None + view_28 = attn_output_10.view(-1, 768) + attn_output_10 = None + x_18 = torch.addmm( + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_, + view_28, + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_ = ( + view_28 + ) = ( + l_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_19 = x_18.view((1, 9, 768)) + x_18 = None + attn_output_11 = torch.nn.functional.dropout(x_19, 0.1, False, False) + x_19 = None + hidden_states_15 = attn_output_11 + hidden_states_13 + attn_output_11 = hidden_states_13 = None + hidden_states_16 = torch.nn.functional.layer_norm( + hidden_states_15, + (768,), + l_self_modules_h_modules_2_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_2_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_2_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_2_modules_ln_2_parameters_bias_ + ) = None + view_30 = hidden_states_16.view(-1, 768) + hidden_states_16 = None + x_20 = torch.addmm( + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_, + view_30, + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_30 + ) = ( + l_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_21 = x_20.view((1, 9, 3072)) + x_20 = None + mul_8 = 0.5 * x_21 + pow_3 = torch.pow(x_21, 3.0) + mul_9 = 0.044715 * pow_3 + pow_3 = None + add_10 = x_21 + mul_9 + x_21 = mul_9 = None + mul_10 = 0.7978845608028654 * add_10 + add_10 = None + tanh_2 = torch.tanh(mul_10) + mul_10 = None + add_11 = 1.0 + tanh_2 + tanh_2 = None + hidden_states_17 = mul_8 * add_11 + mul_8 = add_11 = None + view_32 = hidden_states_17.view(-1, 3072) + hidden_states_17 = None + x_22 = torch.addmm( + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_, + view_32, + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_32 + ) = ( + l_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_23 = x_22.view((1, 9, 768)) + x_22 = None + hidden_states_18 = torch.nn.functional.dropout(x_23, 0.1, False, False) + x_23 = None + hidden_states_19 = hidden_states_15 + hidden_states_18 + hidden_states_15 = hidden_states_18 = None + hidden_states_20 = torch.nn.functional.layer_norm( + hidden_states_19, + (768,), + l_self_modules_h_modules_3_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_3_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_3_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_3_modules_ln_1_parameters_bias_ + ) = None + view_34 = hidden_states_20.view(-1, 768) + hidden_states_20 = None + x_24 = torch.addmm( + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_, + view_34, + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_ = ( + view_34 + ) = ( + l_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_25 = x_24.view((1, 9, 2304)) + x_24 = None + split_3 = x_25.split(768, dim=2) + x_25 = None + query_states_6 = split_3[0] + key_states_6 = split_3[1] + value_states_6 = split_3[2] + split_3 = None + view_36 = key_states_6.view((1, 9, -1, 64)) + key_states_6 = None + key_states_7 = view_36.transpose(1, 2) + view_36 = None + view_37 = value_states_6.view((1, 9, -1, 64)) + value_states_6 = None + value_states_7 = view_37.transpose(1, 2) + view_37 = None + view_38 = query_states_6.view((1, 9, -1, 64)) + query_states_6 = None + query_states_7 = view_38.transpose(1, 2) + view_38 = None + attention_mask_5 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_3 = query_states_7.contiguous() + query_states_7 = None + key_3 = key_states_7.contiguous() + value_3 = value_states_7.contiguous() + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_3, + key_3, + value_3, + attn_mask=attention_mask_5, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_3 = key_3 = value_3 = attention_mask_5 = None + transpose_15 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_13 = transpose_15.contiguous() + transpose_15 = None + reshape_3 = attn_output_13.reshape(1, 9, -1) + attn_output_13 = None + attn_output_14 = reshape_3.contiguous() + reshape_3 = None + view_39 = attn_output_14.view(-1, 768) + attn_output_14 = None + x_26 = torch.addmm( + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_, + view_39, + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_ = ( + view_39 + ) = ( + l_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_27 = x_26.view((1, 9, 768)) + x_26 = None + attn_output_15 = torch.nn.functional.dropout(x_27, 0.1, False, False) + x_27 = None + hidden_states_21 = attn_output_15 + hidden_states_19 + attn_output_15 = hidden_states_19 = None + hidden_states_22 = torch.nn.functional.layer_norm( + hidden_states_21, + (768,), + l_self_modules_h_modules_3_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_3_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_3_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_3_modules_ln_2_parameters_bias_ + ) = None + view_41 = hidden_states_22.view(-1, 768) + hidden_states_22 = None + x_28 = torch.addmm( + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_, + view_41, + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_41 + ) = ( + l_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_29 = x_28.view((1, 9, 3072)) + x_28 = None + mul_12 = 0.5 * x_29 + pow_4 = torch.pow(x_29, 3.0) + mul_13 = 0.044715 * pow_4 + pow_4 = None + add_14 = x_29 + mul_13 + x_29 = mul_13 = None + mul_14 = 0.7978845608028654 * add_14 + add_14 = None + tanh_3 = torch.tanh(mul_14) + mul_14 = None + add_15 = 1.0 + tanh_3 + tanh_3 = None + hidden_states_23 = mul_12 * add_15 + mul_12 = add_15 = None + view_43 = hidden_states_23.view(-1, 3072) + hidden_states_23 = None + x_30 = torch.addmm( + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_, + view_43, + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_43 + ) = ( + l_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_31 = x_30.view((1, 9, 768)) + x_30 = None + hidden_states_24 = torch.nn.functional.dropout(x_31, 0.1, False, False) + x_31 = None + hidden_states_25 = hidden_states_21 + hidden_states_24 + hidden_states_21 = hidden_states_24 = None + hidden_states_26 = torch.nn.functional.layer_norm( + hidden_states_25, + (768,), + l_self_modules_h_modules_4_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_4_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_4_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_4_modules_ln_1_parameters_bias_ + ) = None + view_45 = hidden_states_26.view(-1, 768) + hidden_states_26 = None + x_32 = torch.addmm( + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_, + view_45, + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_ = ( + view_45 + ) = ( + l_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_33 = x_32.view((1, 9, 2304)) + x_32 = None + split_4 = x_33.split(768, dim=2) + x_33 = None + query_states_8 = split_4[0] + key_states_8 = split_4[1] + value_states_8 = split_4[2] + split_4 = None + view_47 = key_states_8.view((1, 9, -1, 64)) + key_states_8 = None + key_states_9 = view_47.transpose(1, 2) + view_47 = None + view_48 = value_states_8.view((1, 9, -1, 64)) + value_states_8 = None + value_states_9 = view_48.transpose(1, 2) + view_48 = None + view_49 = query_states_8.view((1, 9, -1, 64)) + query_states_8 = None + query_states_9 = view_49.transpose(1, 2) + view_49 = None + attention_mask_6 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_4 = query_states_9.contiguous() + query_states_9 = None + key_4 = key_states_9.contiguous() + value_4 = value_states_9.contiguous() + attn_output_16 = torch._C._nn.scaled_dot_product_attention( + query_4, + key_4, + value_4, + attn_mask=attention_mask_6, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_4 = key_4 = value_4 = attention_mask_6 = None + transpose_19 = attn_output_16.transpose(1, 2) + attn_output_16 = None + attn_output_17 = transpose_19.contiguous() + transpose_19 = None + reshape_4 = attn_output_17.reshape(1, 9, -1) + attn_output_17 = None + attn_output_18 = reshape_4.contiguous() + reshape_4 = None + view_50 = attn_output_18.view(-1, 768) + attn_output_18 = None + x_34 = torch.addmm( + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_, + view_50, + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_ = ( + view_50 + ) = ( + l_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_35 = x_34.view((1, 9, 768)) + x_34 = None + attn_output_19 = torch.nn.functional.dropout(x_35, 0.1, False, False) + x_35 = None + hidden_states_27 = attn_output_19 + hidden_states_25 + attn_output_19 = hidden_states_25 = None + hidden_states_28 = torch.nn.functional.layer_norm( + hidden_states_27, + (768,), + l_self_modules_h_modules_4_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_4_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_4_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_4_modules_ln_2_parameters_bias_ + ) = None + view_52 = hidden_states_28.view(-1, 768) + hidden_states_28 = None + x_36 = torch.addmm( + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_, + view_52, + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_52 + ) = ( + l_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_37 = x_36.view((1, 9, 3072)) + x_36 = None + mul_16 = 0.5 * x_37 + pow_5 = torch.pow(x_37, 3.0) + mul_17 = 0.044715 * pow_5 + pow_5 = None + add_18 = x_37 + mul_17 + x_37 = mul_17 = None + mul_18 = 0.7978845608028654 * add_18 + add_18 = None + tanh_4 = torch.tanh(mul_18) + mul_18 = None + add_19 = 1.0 + tanh_4 + tanh_4 = None + hidden_states_29 = mul_16 * add_19 + mul_16 = add_19 = None + view_54 = hidden_states_29.view(-1, 3072) + hidden_states_29 = None + x_38 = torch.addmm( + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_, + view_54, + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_54 + ) = ( + l_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_39 = x_38.view((1, 9, 768)) + x_38 = None + hidden_states_30 = torch.nn.functional.dropout(x_39, 0.1, False, False) + x_39 = None + hidden_states_31 = hidden_states_27 + hidden_states_30 + hidden_states_27 = hidden_states_30 = None + hidden_states_32 = torch.nn.functional.layer_norm( + hidden_states_31, + (768,), + l_self_modules_h_modules_5_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_5_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_5_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_5_modules_ln_1_parameters_bias_ + ) = None + view_56 = hidden_states_32.view(-1, 768) + hidden_states_32 = None + x_40 = torch.addmm( + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_, + view_56, + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_ = ( + view_56 + ) = ( + l_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_41 = x_40.view((1, 9, 2304)) + x_40 = None + split_5 = x_41.split(768, dim=2) + x_41 = None + query_states_10 = split_5[0] + key_states_10 = split_5[1] + value_states_10 = split_5[2] + split_5 = None + view_58 = key_states_10.view((1, 9, -1, 64)) + key_states_10 = None + key_states_11 = view_58.transpose(1, 2) + view_58 = None + view_59 = value_states_10.view((1, 9, -1, 64)) + value_states_10 = None + value_states_11 = view_59.transpose(1, 2) + view_59 = None + view_60 = query_states_10.view((1, 9, -1, 64)) + query_states_10 = None + query_states_11 = view_60.transpose(1, 2) + view_60 = None + attention_mask_7 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_5 = query_states_11.contiguous() + query_states_11 = None + key_5 = key_states_11.contiguous() + value_5 = value_states_11.contiguous() + attn_output_20 = torch._C._nn.scaled_dot_product_attention( + query_5, + key_5, + value_5, + attn_mask=attention_mask_7, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_5 = key_5 = value_5 = attention_mask_7 = None + transpose_23 = attn_output_20.transpose(1, 2) + attn_output_20 = None + attn_output_21 = transpose_23.contiguous() + transpose_23 = None + reshape_5 = attn_output_21.reshape(1, 9, -1) + attn_output_21 = None + attn_output_22 = reshape_5.contiguous() + reshape_5 = None + view_61 = attn_output_22.view(-1, 768) + attn_output_22 = None + x_42 = torch.addmm( + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_, + view_61, + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_ = ( + view_61 + ) = ( + l_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_43 = x_42.view((1, 9, 768)) + x_42 = None + attn_output_23 = torch.nn.functional.dropout(x_43, 0.1, False, False) + x_43 = None + hidden_states_33 = attn_output_23 + hidden_states_31 + attn_output_23 = hidden_states_31 = None + hidden_states_34 = torch.nn.functional.layer_norm( + hidden_states_33, + (768,), + l_self_modules_h_modules_5_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_5_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_5_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_5_modules_ln_2_parameters_bias_ + ) = None + view_63 = hidden_states_34.view(-1, 768) + hidden_states_34 = None + x_44 = torch.addmm( + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_, + view_63, + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_63 + ) = ( + l_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_45 = x_44.view((1, 9, 3072)) + x_44 = None + mul_20 = 0.5 * x_45 + pow_6 = torch.pow(x_45, 3.0) + mul_21 = 0.044715 * pow_6 + pow_6 = None + add_22 = x_45 + mul_21 + x_45 = mul_21 = None + mul_22 = 0.7978845608028654 * add_22 + add_22 = None + tanh_5 = torch.tanh(mul_22) + mul_22 = None + add_23 = 1.0 + tanh_5 + tanh_5 = None + hidden_states_35 = mul_20 * add_23 + mul_20 = add_23 = None + view_65 = hidden_states_35.view(-1, 3072) + hidden_states_35 = None + x_46 = torch.addmm( + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_, + view_65, + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_65 + ) = ( + l_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_47 = x_46.view((1, 9, 768)) + x_46 = None + hidden_states_36 = torch.nn.functional.dropout(x_47, 0.1, False, False) + x_47 = None + hidden_states_37 = hidden_states_33 + hidden_states_36 + hidden_states_33 = hidden_states_36 = None + hidden_states_38 = torch.nn.functional.layer_norm( + hidden_states_37, + (768,), + l_self_modules_h_modules_6_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_6_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_6_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_6_modules_ln_1_parameters_bias_ + ) = None + view_67 = hidden_states_38.view(-1, 768) + hidden_states_38 = None + x_48 = torch.addmm( + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_, + view_67, + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_ = ( + view_67 + ) = ( + l_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_49 = x_48.view((1, 9, 2304)) + x_48 = None + split_6 = x_49.split(768, dim=2) + x_49 = None + query_states_12 = split_6[0] + key_states_12 = split_6[1] + value_states_12 = split_6[2] + split_6 = None + view_69 = key_states_12.view((1, 9, -1, 64)) + key_states_12 = None + key_states_13 = view_69.transpose(1, 2) + view_69 = None + view_70 = value_states_12.view((1, 9, -1, 64)) + value_states_12 = None + value_states_13 = view_70.transpose(1, 2) + view_70 = None + view_71 = query_states_12.view((1, 9, -1, 64)) + query_states_12 = None + query_states_13 = view_71.transpose(1, 2) + view_71 = None + attention_mask_8 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_6 = query_states_13.contiguous() + query_states_13 = None + key_6 = key_states_13.contiguous() + value_6 = value_states_13.contiguous() + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_6, + key_6, + value_6, + attn_mask=attention_mask_8, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_6 = key_6 = value_6 = attention_mask_8 = None + transpose_27 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_25 = transpose_27.contiguous() + transpose_27 = None + reshape_6 = attn_output_25.reshape(1, 9, -1) + attn_output_25 = None + attn_output_26 = reshape_6.contiguous() + reshape_6 = None + view_72 = attn_output_26.view(-1, 768) + attn_output_26 = None + x_50 = torch.addmm( + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_, + view_72, + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_ = ( + view_72 + ) = ( + l_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_51 = x_50.view((1, 9, 768)) + x_50 = None + attn_output_27 = torch.nn.functional.dropout(x_51, 0.1, False, False) + x_51 = None + hidden_states_39 = attn_output_27 + hidden_states_37 + attn_output_27 = hidden_states_37 = None + hidden_states_40 = torch.nn.functional.layer_norm( + hidden_states_39, + (768,), + l_self_modules_h_modules_6_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_6_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_6_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_6_modules_ln_2_parameters_bias_ + ) = None + view_74 = hidden_states_40.view(-1, 768) + hidden_states_40 = None + x_52 = torch.addmm( + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_, + view_74, + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_74 + ) = ( + l_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_53 = x_52.view((1, 9, 3072)) + x_52 = None + mul_24 = 0.5 * x_53 + pow_7 = torch.pow(x_53, 3.0) + mul_25 = 0.044715 * pow_7 + pow_7 = None + add_26 = x_53 + mul_25 + x_53 = mul_25 = None + mul_26 = 0.7978845608028654 * add_26 + add_26 = None + tanh_6 = torch.tanh(mul_26) + mul_26 = None + add_27 = 1.0 + tanh_6 + tanh_6 = None + hidden_states_41 = mul_24 * add_27 + mul_24 = add_27 = None + view_76 = hidden_states_41.view(-1, 3072) + hidden_states_41 = None + x_54 = torch.addmm( + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_, + view_76, + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_76 + ) = ( + l_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_55 = x_54.view((1, 9, 768)) + x_54 = None + hidden_states_42 = torch.nn.functional.dropout(x_55, 0.1, False, False) + x_55 = None + hidden_states_43 = hidden_states_39 + hidden_states_42 + hidden_states_39 = hidden_states_42 = None + hidden_states_44 = torch.nn.functional.layer_norm( + hidden_states_43, + (768,), + l_self_modules_h_modules_7_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_7_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_7_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_7_modules_ln_1_parameters_bias_ + ) = None + view_78 = hidden_states_44.view(-1, 768) + hidden_states_44 = None + x_56 = torch.addmm( + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_, + view_78, + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_ = ( + view_78 + ) = ( + l_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_57 = x_56.view((1, 9, 2304)) + x_56 = None + split_7 = x_57.split(768, dim=2) + x_57 = None + query_states_14 = split_7[0] + key_states_14 = split_7[1] + value_states_14 = split_7[2] + split_7 = None + view_80 = key_states_14.view((1, 9, -1, 64)) + key_states_14 = None + key_states_15 = view_80.transpose(1, 2) + view_80 = None + view_81 = value_states_14.view((1, 9, -1, 64)) + value_states_14 = None + value_states_15 = view_81.transpose(1, 2) + view_81 = None + view_82 = query_states_14.view((1, 9, -1, 64)) + query_states_14 = None + query_states_15 = view_82.transpose(1, 2) + view_82 = None + attention_mask_9 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_7 = query_states_15.contiguous() + query_states_15 = None + key_7 = key_states_15.contiguous() + value_7 = value_states_15.contiguous() + attn_output_28 = torch._C._nn.scaled_dot_product_attention( + query_7, + key_7, + value_7, + attn_mask=attention_mask_9, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_7 = key_7 = value_7 = attention_mask_9 = None + transpose_31 = attn_output_28.transpose(1, 2) + attn_output_28 = None + attn_output_29 = transpose_31.contiguous() + transpose_31 = None + reshape_7 = attn_output_29.reshape(1, 9, -1) + attn_output_29 = None + attn_output_30 = reshape_7.contiguous() + reshape_7 = None + view_83 = attn_output_30.view(-1, 768) + attn_output_30 = None + x_58 = torch.addmm( + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_, + view_83, + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_ = ( + view_83 + ) = ( + l_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_59 = x_58.view((1, 9, 768)) + x_58 = None + attn_output_31 = torch.nn.functional.dropout(x_59, 0.1, False, False) + x_59 = None + hidden_states_45 = attn_output_31 + hidden_states_43 + attn_output_31 = hidden_states_43 = None + hidden_states_46 = torch.nn.functional.layer_norm( + hidden_states_45, + (768,), + l_self_modules_h_modules_7_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_7_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_7_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_7_modules_ln_2_parameters_bias_ + ) = None + view_85 = hidden_states_46.view(-1, 768) + hidden_states_46 = None + x_60 = torch.addmm( + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_, + view_85, + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_85 + ) = ( + l_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_61 = x_60.view((1, 9, 3072)) + x_60 = None + mul_28 = 0.5 * x_61 + pow_8 = torch.pow(x_61, 3.0) + mul_29 = 0.044715 * pow_8 + pow_8 = None + add_30 = x_61 + mul_29 + x_61 = mul_29 = None + mul_30 = 0.7978845608028654 * add_30 + add_30 = None + tanh_7 = torch.tanh(mul_30) + mul_30 = None + add_31 = 1.0 + tanh_7 + tanh_7 = None + hidden_states_47 = mul_28 * add_31 + mul_28 = add_31 = None + view_87 = hidden_states_47.view(-1, 3072) + hidden_states_47 = None + x_62 = torch.addmm( + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_, + view_87, + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_87 + ) = ( + l_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_63 = x_62.view((1, 9, 768)) + x_62 = None + hidden_states_48 = torch.nn.functional.dropout(x_63, 0.1, False, False) + x_63 = None + hidden_states_49 = hidden_states_45 + hidden_states_48 + hidden_states_45 = hidden_states_48 = None + hidden_states_50 = torch.nn.functional.layer_norm( + hidden_states_49, + (768,), + l_self_modules_h_modules_8_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_8_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_8_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_8_modules_ln_1_parameters_bias_ + ) = None + view_89 = hidden_states_50.view(-1, 768) + hidden_states_50 = None + x_64 = torch.addmm( + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_, + view_89, + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_ = ( + view_89 + ) = ( + l_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_65 = x_64.view((1, 9, 2304)) + x_64 = None + split_8 = x_65.split(768, dim=2) + x_65 = None + query_states_16 = split_8[0] + key_states_16 = split_8[1] + value_states_16 = split_8[2] + split_8 = None + view_91 = key_states_16.view((1, 9, -1, 64)) + key_states_16 = None + key_states_17 = view_91.transpose(1, 2) + view_91 = None + view_92 = value_states_16.view((1, 9, -1, 64)) + value_states_16 = None + value_states_17 = view_92.transpose(1, 2) + view_92 = None + view_93 = query_states_16.view((1, 9, -1, 64)) + query_states_16 = None + query_states_17 = view_93.transpose(1, 2) + view_93 = None + attention_mask_10 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_8 = query_states_17.contiguous() + query_states_17 = None + key_8 = key_states_17.contiguous() + value_8 = value_states_17.contiguous() + attn_output_32 = torch._C._nn.scaled_dot_product_attention( + query_8, + key_8, + value_8, + attn_mask=attention_mask_10, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_8 = key_8 = value_8 = attention_mask_10 = None + transpose_35 = attn_output_32.transpose(1, 2) + attn_output_32 = None + attn_output_33 = transpose_35.contiguous() + transpose_35 = None + reshape_8 = attn_output_33.reshape(1, 9, -1) + attn_output_33 = None + attn_output_34 = reshape_8.contiguous() + reshape_8 = None + view_94 = attn_output_34.view(-1, 768) + attn_output_34 = None + x_66 = torch.addmm( + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_, + view_94, + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_ = ( + view_94 + ) = ( + l_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_67 = x_66.view((1, 9, 768)) + x_66 = None + attn_output_35 = torch.nn.functional.dropout(x_67, 0.1, False, False) + x_67 = None + hidden_states_51 = attn_output_35 + hidden_states_49 + attn_output_35 = hidden_states_49 = None + hidden_states_52 = torch.nn.functional.layer_norm( + hidden_states_51, + (768,), + l_self_modules_h_modules_8_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_8_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_8_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_8_modules_ln_2_parameters_bias_ + ) = None + view_96 = hidden_states_52.view(-1, 768) + hidden_states_52 = None + x_68 = torch.addmm( + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_, + view_96, + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_96 + ) = ( + l_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_69 = x_68.view((1, 9, 3072)) + x_68 = None + mul_32 = 0.5 * x_69 + pow_9 = torch.pow(x_69, 3.0) + mul_33 = 0.044715 * pow_9 + pow_9 = None + add_34 = x_69 + mul_33 + x_69 = mul_33 = None + mul_34 = 0.7978845608028654 * add_34 + add_34 = None + tanh_8 = torch.tanh(mul_34) + mul_34 = None + add_35 = 1.0 + tanh_8 + tanh_8 = None + hidden_states_53 = mul_32 * add_35 + mul_32 = add_35 = None + view_98 = hidden_states_53.view(-1, 3072) + hidden_states_53 = None + x_70 = torch.addmm( + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_, + view_98, + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_98 + ) = ( + l_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_71 = x_70.view((1, 9, 768)) + x_70 = None + hidden_states_54 = torch.nn.functional.dropout(x_71, 0.1, False, False) + x_71 = None + hidden_states_55 = hidden_states_51 + hidden_states_54 + hidden_states_51 = hidden_states_54 = None + hidden_states_56 = torch.nn.functional.layer_norm( + hidden_states_55, + (768,), + l_self_modules_h_modules_9_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_9_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_9_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_9_modules_ln_1_parameters_bias_ + ) = None + view_100 = hidden_states_56.view(-1, 768) + hidden_states_56 = None + x_72 = torch.addmm( + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_, + view_100, + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_ = ( + view_100 + ) = ( + l_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_73 = x_72.view((1, 9, 2304)) + x_72 = None + split_9 = x_73.split(768, dim=2) + x_73 = None + query_states_18 = split_9[0] + key_states_18 = split_9[1] + value_states_18 = split_9[2] + split_9 = None + view_102 = key_states_18.view((1, 9, -1, 64)) + key_states_18 = None + key_states_19 = view_102.transpose(1, 2) + view_102 = None + view_103 = value_states_18.view((1, 9, -1, 64)) + value_states_18 = None + value_states_19 = view_103.transpose(1, 2) + view_103 = None + view_104 = query_states_18.view((1, 9, -1, 64)) + query_states_18 = None + query_states_19 = view_104.transpose(1, 2) + view_104 = None + attention_mask_11 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_9 = query_states_19.contiguous() + query_states_19 = None + key_9 = key_states_19.contiguous() + value_9 = value_states_19.contiguous() + attn_output_36 = torch._C._nn.scaled_dot_product_attention( + query_9, + key_9, + value_9, + attn_mask=attention_mask_11, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_9 = key_9 = value_9 = attention_mask_11 = None + transpose_39 = attn_output_36.transpose(1, 2) + attn_output_36 = None + attn_output_37 = transpose_39.contiguous() + transpose_39 = None + reshape_9 = attn_output_37.reshape(1, 9, -1) + attn_output_37 = None + attn_output_38 = reshape_9.contiguous() + reshape_9 = None + view_105 = attn_output_38.view(-1, 768) + attn_output_38 = None + x_74 = torch.addmm( + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_, + view_105, + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_ = ( + view_105 + ) = ( + l_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_75 = x_74.view((1, 9, 768)) + x_74 = None + attn_output_39 = torch.nn.functional.dropout(x_75, 0.1, False, False) + x_75 = None + hidden_states_57 = attn_output_39 + hidden_states_55 + attn_output_39 = hidden_states_55 = None + hidden_states_58 = torch.nn.functional.layer_norm( + hidden_states_57, + (768,), + l_self_modules_h_modules_9_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_9_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_9_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_9_modules_ln_2_parameters_bias_ + ) = None + view_107 = hidden_states_58.view(-1, 768) + hidden_states_58 = None + x_76 = torch.addmm( + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_, + view_107, + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_107 + ) = ( + l_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_77 = x_76.view((1, 9, 3072)) + x_76 = None + mul_36 = 0.5 * x_77 + pow_10 = torch.pow(x_77, 3.0) + mul_37 = 0.044715 * pow_10 + pow_10 = None + add_38 = x_77 + mul_37 + x_77 = mul_37 = None + mul_38 = 0.7978845608028654 * add_38 + add_38 = None + tanh_9 = torch.tanh(mul_38) + mul_38 = None + add_39 = 1.0 + tanh_9 + tanh_9 = None + hidden_states_59 = mul_36 * add_39 + mul_36 = add_39 = None + view_109 = hidden_states_59.view(-1, 3072) + hidden_states_59 = None + x_78 = torch.addmm( + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_, + view_109, + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_109 + ) = ( + l_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_79 = x_78.view((1, 9, 768)) + x_78 = None + hidden_states_60 = torch.nn.functional.dropout(x_79, 0.1, False, False) + x_79 = None + hidden_states_61 = hidden_states_57 + hidden_states_60 + hidden_states_57 = hidden_states_60 = None + hidden_states_62 = torch.nn.functional.layer_norm( + hidden_states_61, + (768,), + l_self_modules_h_modules_10_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_10_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_10_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_10_modules_ln_1_parameters_bias_ + ) = None + view_111 = hidden_states_62.view(-1, 768) + hidden_states_62 = None + x_80 = torch.addmm( + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_, + view_111, + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_ = ( + view_111 + ) = ( + l_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_81 = x_80.view((1, 9, 2304)) + x_80 = None + split_10 = x_81.split(768, dim=2) + x_81 = None + query_states_20 = split_10[0] + key_states_20 = split_10[1] + value_states_20 = split_10[2] + split_10 = None + view_113 = key_states_20.view((1, 9, -1, 64)) + key_states_20 = None + key_states_21 = view_113.transpose(1, 2) + view_113 = None + view_114 = value_states_20.view((1, 9, -1, 64)) + value_states_20 = None + value_states_21 = view_114.transpose(1, 2) + view_114 = None + view_115 = query_states_20.view((1, 9, -1, 64)) + query_states_20 = None + query_states_21 = view_115.transpose(1, 2) + view_115 = None + attention_mask_12 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + query_10 = query_states_21.contiguous() + query_states_21 = None + key_10 = key_states_21.contiguous() + value_10 = value_states_21.contiguous() + attn_output_40 = torch._C._nn.scaled_dot_product_attention( + query_10, + key_10, + value_10, + attn_mask=attention_mask_12, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_10 = key_10 = value_10 = attention_mask_12 = None + transpose_43 = attn_output_40.transpose(1, 2) + attn_output_40 = None + attn_output_41 = transpose_43.contiguous() + transpose_43 = None + reshape_10 = attn_output_41.reshape(1, 9, -1) + attn_output_41 = None + attn_output_42 = reshape_10.contiguous() + reshape_10 = None + view_116 = attn_output_42.view(-1, 768) + attn_output_42 = None + x_82 = torch.addmm( + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_, + view_116, + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_ = ( + view_116 + ) = ( + l_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_83 = x_82.view((1, 9, 768)) + x_82 = None + attn_output_43 = torch.nn.functional.dropout(x_83, 0.1, False, False) + x_83 = None + hidden_states_63 = attn_output_43 + hidden_states_61 + attn_output_43 = hidden_states_61 = None + hidden_states_64 = torch.nn.functional.layer_norm( + hidden_states_63, + (768,), + l_self_modules_h_modules_10_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_10_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_10_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_10_modules_ln_2_parameters_bias_ + ) = None + view_118 = hidden_states_64.view(-1, 768) + hidden_states_64 = None + x_84 = torch.addmm( + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_, + view_118, + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_118 + ) = ( + l_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_85 = x_84.view((1, 9, 3072)) + x_84 = None + mul_40 = 0.5 * x_85 + pow_11 = torch.pow(x_85, 3.0) + mul_41 = 0.044715 * pow_11 + pow_11 = None + add_42 = x_85 + mul_41 + x_85 = mul_41 = None + mul_42 = 0.7978845608028654 * add_42 + add_42 = None + tanh_10 = torch.tanh(mul_42) + mul_42 = None + add_43 = 1.0 + tanh_10 + tanh_10 = None + hidden_states_65 = mul_40 * add_43 + mul_40 = add_43 = None + view_120 = hidden_states_65.view(-1, 3072) + hidden_states_65 = None + x_86 = torch.addmm( + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_, + view_120, + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_120 + ) = ( + l_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_87 = x_86.view((1, 9, 768)) + x_86 = None + hidden_states_66 = torch.nn.functional.dropout(x_87, 0.1, False, False) + x_87 = None + hidden_states_67 = hidden_states_63 + hidden_states_66 + hidden_states_63 = hidden_states_66 = None + hidden_states_68 = torch.nn.functional.layer_norm( + hidden_states_67, + (768,), + l_self_modules_h_modules_11_modules_ln_1_parameters_weight_, + l_self_modules_h_modules_11_modules_ln_1_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_11_modules_ln_1_parameters_weight_ = ( + l_self_modules_h_modules_11_modules_ln_1_parameters_bias_ + ) = None + view_122 = hidden_states_68.view(-1, 768) + hidden_states_68 = None + x_88 = torch.addmm( + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_, + view_122, + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_, + ) + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_ = ( + view_122 + ) = ( + l_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_ + ) = None + x_89 = x_88.view((1, 9, 2304)) + x_88 = None + split_11 = x_89.split(768, dim=2) + x_89 = None + query_states_22 = split_11[0] + key_states_22 = split_11[1] + value_states_22 = split_11[2] + split_11 = None + view_124 = key_states_22.view((1, 9, -1, 64)) + key_states_22 = None + key_states_23 = view_124.transpose(1, 2) + view_124 = None + view_125 = value_states_22.view((1, 9, -1, 64)) + value_states_22 = None + value_states_23 = view_125.transpose(1, 2) + view_125 = None + view_126 = query_states_22.view((1, 9, -1, 64)) + query_states_22 = None + query_states_23 = view_126.transpose(1, 2) + view_126 = None + attention_mask_13 = causal_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 9, None), + ) + ] + causal_mask = None + query_11 = query_states_23.contiguous() + query_states_23 = None + key_11 = key_states_23.contiguous() + value_11 = value_states_23.contiguous() + attn_output_44 = torch._C._nn.scaled_dot_product_attention( + query_11, + key_11, + value_11, + attn_mask=attention_mask_13, + dropout_p=0.0, + scale=None, + is_causal=False, + ) + query_11 = key_11 = value_11 = attention_mask_13 = None + transpose_47 = attn_output_44.transpose(1, 2) + attn_output_44 = None + attn_output_45 = transpose_47.contiguous() + transpose_47 = None + reshape_11 = attn_output_45.reshape(1, 9, -1) + attn_output_45 = None + attn_output_46 = reshape_11.contiguous() + reshape_11 = None + view_127 = attn_output_46.view(-1, 768) + attn_output_46 = None + x_90 = torch.addmm( + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_, + view_127, + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_ = ( + view_127 + ) = ( + l_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_ + ) = None + x_91 = x_90.view((1, 9, 768)) + x_90 = None + attn_output_47 = torch.nn.functional.dropout(x_91, 0.1, False, False) + x_91 = None + hidden_states_69 = attn_output_47 + hidden_states_67 + attn_output_47 = hidden_states_67 = None + hidden_states_70 = torch.nn.functional.layer_norm( + hidden_states_69, + (768,), + l_self_modules_h_modules_11_modules_ln_2_parameters_weight_, + l_self_modules_h_modules_11_modules_ln_2_parameters_bias_, + 1e-05, + ) + l_self_modules_h_modules_11_modules_ln_2_parameters_weight_ = ( + l_self_modules_h_modules_11_modules_ln_2_parameters_bias_ + ) = None + view_129 = hidden_states_70.view(-1, 768) + hidden_states_70 = None + x_92 = torch.addmm( + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_, + view_129, + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_, + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_ = ( + view_129 + ) = ( + l_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_ + ) = None + x_93 = x_92.view((1, 9, 3072)) + x_92 = None + mul_44 = 0.5 * x_93 + pow_12 = torch.pow(x_93, 3.0) + mul_45 = 0.044715 * pow_12 + pow_12 = None + add_46 = x_93 + mul_45 + x_93 = mul_45 = None + mul_46 = 0.7978845608028654 * add_46 + add_46 = None + tanh_11 = torch.tanh(mul_46) + mul_46 = None + add_47 = 1.0 + tanh_11 + tanh_11 = None + hidden_states_71 = mul_44 * add_47 + mul_44 = add_47 = None + view_131 = hidden_states_71.view(-1, 3072) + hidden_states_71 = None + x_94 = torch.addmm( + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_, + view_131, + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_, + ) + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_ = ( + view_131 + ) = ( + l_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_ + ) = None + x_95 = x_94.view((1, 9, 768)) + x_94 = None + hidden_states_72 = torch.nn.functional.dropout(x_95, 0.1, False, False) + x_95 = None + hidden_states_73 = hidden_states_69 + hidden_states_72 + hidden_states_69 = hidden_states_72 = None + hidden_states_74 = torch.nn.functional.layer_norm( + hidden_states_73, + (768,), + l_self_modules_ln_f_parameters_weight_, + l_self_modules_ln_f_parameters_bias_, + 1e-05, + ) + hidden_states_73 = ( + l_self_modules_ln_f_parameters_weight_ + ) = l_self_modules_ln_f_parameters_bias_ = None + hidden_states_75 = hidden_states_74.view((-1, 9, 768)) + hidden_states_74 = None + return ( + value_states_1, + key_states_1, + value_states_3, + key_states_3, + value_states_5, + key_states_5, + value_states_7, + key_states_7, + value_states_9, + key_states_9, + value_states_11, + key_states_11, + value_states_13, + key_states_13, + value_states_15, + key_states_15, + value_states_17, + key_states_17, + value_states_19, + key_states_19, + value_states_21, + key_states_21, + value_states_23, + key_states_23, + hidden_states_75, + ) diff --git a/samples/transformers-auto-model/gpt2/weight_meta.py b/samples/transformers-auto-model/gpt2/weight_meta.py new file mode 100644 index 0000000000..5a99ad13db --- /dev/null +++ b/samples/transformers-auto-model/gpt2/weight_meta.py @@ -0,0 +1,1498 @@ +class Program_weight_tensor_meta_L_self_modules_wte_parameters_weight_: + name = "L_self_modules_wte_parameters_weight_" + shape = [50257, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.144 + data = None + + +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 9] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [37065, 7934, 318, 973, 284, 7925, 31350, 28770, 13] + + +class Program_weight_tensor_meta_L_self_modules_wpe_parameters_weight_: + name = "L_self_modules_wpe_parameters_weight_" + shape = [1024, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 9] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.180 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.226 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.200 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.259 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.147 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.868 + std = 0.485 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.009 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.132 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.102 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_0_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.088 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.223 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.211 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.140 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.105 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.102 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.243 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.072 + std = 0.095 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.131 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.100 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_1_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.241 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.165 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.153 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.293 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.134 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.112 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_2_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.301 + std = 0.054 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.142 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.142 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.108 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.084 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.307 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.010 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.093 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_3_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.319 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.005 + std = 0.240 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.100 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.273 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.086 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.137 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_4_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.373 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.111 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.279 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.008 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.085 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.106 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_5_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.097 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.346 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.012 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.106 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.259 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.086 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.126 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.121 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_6_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.357 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.014 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.138 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.129 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.256 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.009 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.088 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.126 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.129 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_7_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.119 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.335 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.013 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.131 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.140 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.122 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.257 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.085 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_8_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.358 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.016 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.141 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.126 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.209 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.137 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.265 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.084 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.159 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_9_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.156 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.378 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.019 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.145 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.232 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.147 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.290 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.021 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.077 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.194 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_10_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.178 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_ln_1_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_ln_1_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.479 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_ln_1_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_ln_1_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.023 + std = 0.057 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_bias_" + shape = [2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.121 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_attn_modules_c_attn_parameters_weight_" + shape = [768, 2304] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.022 + std = 0.469 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_attn_modules_c_proj_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.182 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_ln_2_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_ln_2_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.504 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_ln_2_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_ln_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.009 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.064 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_mlp_modules_c_fc_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.130 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_: + name = "L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.108 + data = None + + +class Program_weight_tensor_meta_L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_: + name = "L_self_modules_h_modules_11_modules_mlp_modules_c_proj_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.198 + data = None + + +class Program_weight_tensor_meta_L_self_modules_ln_f_parameters_weight_: + name = "L_self_modules_ln_f_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.508 + std = 1.391 + data = None + + +class Program_weight_tensor_meta_L_self_modules_ln_f_parameters_bias_: + name = "L_self_modules_ln_f_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.420 + data = None diff --git a/samples/transformers-auto-model/roberta-base/graph_hash.txt b/samples/transformers-auto-model/roberta-base/graph_hash.txt new file mode 100644 index 0000000000..95ced5af14 --- /dev/null +++ b/samples/transformers-auto-model/roberta-base/graph_hash.txt @@ -0,0 +1 @@ +459b0ae40a151771b060dad5cd1494c50aa2d9237b500365bdc97ff30df27aed \ No newline at end of file diff --git a/samples/transformers-auto-model/roberta-base/graph_net.json b/samples/transformers-auto-model/roberta-base/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/roberta-base/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/roberta-base/input_meta.py b/samples/transformers-auto-model/roberta-base/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/roberta-base/input_tensor_constraints.py b/samples/transformers-auto-model/roberta-base/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/roberta-base/model.py b/samples/transformers-auto-model/roberta-base/model.py new file mode 100644 index 0000000000..03c7c08ebf --- /dev/null +++ b/samples/transformers-auto-model/roberta-base/model.py @@ -0,0 +1,1618 @@ +import torch + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_self_modules_embeddings_buffers_token_type_ids_: torch.Tensor, + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_attention_mask_: torch.Tensor, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_pooler_modules_dense_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_self_modules_embeddings_buffers_token_type_ids_ = ( + L_self_modules_embeddings_buffers_token_type_ids_ + ) + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ = ( + L_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_weight_ + ) + l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = ( + L_self_modules_embeddings_modules_LayerNorm_parameters_bias_ + ) + l_attention_mask_ = L_attention_mask_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_ + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_ + l_self_modules_pooler_modules_dense_parameters_weight_ = ( + L_self_modules_pooler_modules_dense_parameters_weight_ + ) + l_self_modules_pooler_modules_dense_parameters_bias_ = ( + L_self_modules_pooler_modules_dense_parameters_bias_ + ) + buffered_token_type_ids = l_self_modules_embeddings_buffers_token_type_ids_[ + (slice(None, None, None), slice(None, 11, None)) + ] + l_self_modules_embeddings_buffers_token_type_ids_ = None + buffered_token_type_ids_expanded = buffered_token_type_ids.expand(1, 11) + buffered_token_type_ids = None + ne = l_input_ids_.ne(1) + mask = ne.int() + ne = None + cumsum = torch.cumsum(mask, dim=1) + type_as = cumsum.type_as(mask) + cumsum = None + add = type_as + 0 + type_as = None + incremental_indices = add * mask + add = mask = None + long = incremental_indices.long() + incremental_indices = None + position_ids = long + 1 + long = None + inputs_embeds = torch.nn.functional.embedding( + l_input_ids_, + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_, + 1, + None, + 2.0, + False, + False, + ) + l_input_ids_ = ( + l_self_modules_embeddings_modules_word_embeddings_parameters_weight_ + ) = None + token_type_embeddings = torch.nn.functional.embedding( + buffered_token_type_ids_expanded, + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + buffered_token_type_ids_expanded = ( + l_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_ + ) = None + embeddings = inputs_embeds + token_type_embeddings + inputs_embeds = token_type_embeddings = None + position_embeddings = torch.nn.functional.embedding( + position_ids, + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_, + 1, + None, + 2.0, + False, + False, + ) + position_ids = ( + l_self_modules_embeddings_modules_position_embeddings_parameters_weight_ + ) = None + embeddings += position_embeddings + embeddings_1 = embeddings + embeddings = position_embeddings = None + embeddings_2 = torch.nn.functional.layer_norm( + embeddings_1, + (768,), + l_self_modules_embeddings_modules_layer_norm_parameters_weight_, + l_self_modules_embeddings_modules_layer_norm_parameters_bias_, + 1e-05, + ) + embeddings_1 = ( + l_self_modules_embeddings_modules_layer_norm_parameters_weight_ + ) = l_self_modules_embeddings_modules_layer_norm_parameters_bias_ = None + embeddings_3 = torch.nn.functional.dropout(embeddings_2, 0.1, False, False) + embeddings_2 = None + getitem_1 = l_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_attention_mask_ = None + expand_1 = getitem_1.expand(1, 1, 11, 11) + getitem_1 = None + expanded_mask = expand_1.to(torch.float32) + expand_1 = None + tensor = torch.tensor(1.0, dtype=torch.float32) + inverted_mask = tensor - expanded_mask + tensor = expanded_mask = None + to_1 = inverted_mask.to(torch.bool) + extended_attention_mask = inverted_mask.masked_fill( + to_1, -3.4028234663852886e38 + ) + inverted_mask = to_1 = None + linear = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view = linear.view(1, -1, 12, 64) + linear = None + query_layer = view.transpose(1, 2) + view = None + linear_1 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_1 = linear_1.view(1, -1, 12, 64) + linear_1 = None + key_layer = view_1.transpose(1, 2) + view_1 = None + linear_2 = torch._C._nn.linear( + embeddings_3, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_2 = linear_2.view(1, -1, 12, 64) + linear_2 = None + value_layer = view_2.transpose(1, 2) + view_2 = None + attn_output = torch._C._nn.scaled_dot_product_attention( + query_layer, + key_layer, + value_layer, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer = key_layer = value_layer = None + attn_output_1 = attn_output.transpose(1, 2) + attn_output = None + attn_output_2 = attn_output_1.reshape(1, 11, 768) + attn_output_1 = None + hidden_states = torch._C._nn.linear( + attn_output_2, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_2 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_1 = torch.nn.functional.dropout(hidden_states, 0.1, False, False) + hidden_states = None + add_3 = hidden_states_1 + embeddings_3 + hidden_states_1 = embeddings_3 = None + hidden_states_2 = torch.nn.functional.layer_norm( + add_3, + (768,), + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_3 = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_3 = torch._C._nn.linear( + hidden_states_2, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_4 = torch._C._nn.gelu(hidden_states_3) + hidden_states_3 = None + hidden_states_5 = torch._C._nn.linear( + hidden_states_4, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_4 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_6 = torch.nn.functional.dropout( + hidden_states_5, 0.1, False, False + ) + hidden_states_5 = None + add_4 = hidden_states_6 + hidden_states_2 + hidden_states_6 = hidden_states_2 = None + hidden_states_7 = torch.nn.functional.layer_norm( + add_4, + (768,), + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_4 = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_0_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_6 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_3 = linear_6.view(1, -1, 12, 64) + linear_6 = None + query_layer_1 = view_3.transpose(1, 2) + view_3 = None + linear_7 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_4 = linear_7.view(1, -1, 12, 64) + linear_7 = None + key_layer_1 = view_4.transpose(1, 2) + view_4 = None + linear_8 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_5 = linear_8.view(1, -1, 12, 64) + linear_8 = None + value_layer_1 = view_5.transpose(1, 2) + view_5 = None + attn_output_3 = torch._C._nn.scaled_dot_product_attention( + query_layer_1, + key_layer_1, + value_layer_1, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_1 = key_layer_1 = value_layer_1 = None + attn_output_4 = attn_output_3.transpose(1, 2) + attn_output_3 = None + attn_output_5 = attn_output_4.reshape(1, 11, 768) + attn_output_4 = None + hidden_states_8 = torch._C._nn.linear( + attn_output_5, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_9 = torch.nn.functional.dropout( + hidden_states_8, 0.1, False, False + ) + hidden_states_8 = None + add_5 = hidden_states_9 + hidden_states_7 + hidden_states_9 = hidden_states_7 = None + hidden_states_10 = torch.nn.functional.layer_norm( + add_5, + (768,), + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_5 = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_11 = torch._C._nn.linear( + hidden_states_10, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_12 = torch._C._nn.gelu(hidden_states_11) + hidden_states_11 = None + hidden_states_13 = torch._C._nn.linear( + hidden_states_12, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_12 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_14 = torch.nn.functional.dropout( + hidden_states_13, 0.1, False, False + ) + hidden_states_13 = None + add_6 = hidden_states_14 + hidden_states_10 + hidden_states_14 = hidden_states_10 = None + hidden_states_15 = torch.nn.functional.layer_norm( + add_6, + (768,), + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_6 = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_1_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_12 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_6 = linear_12.view(1, -1, 12, 64) + linear_12 = None + query_layer_2 = view_6.transpose(1, 2) + view_6 = None + linear_13 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_7 = linear_13.view(1, -1, 12, 64) + linear_13 = None + key_layer_2 = view_7.transpose(1, 2) + view_7 = None + linear_14 = torch._C._nn.linear( + hidden_states_15, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_8 = linear_14.view(1, -1, 12, 64) + linear_14 = None + value_layer_2 = view_8.transpose(1, 2) + view_8 = None + attn_output_6 = torch._C._nn.scaled_dot_product_attention( + query_layer_2, + key_layer_2, + value_layer_2, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_2 = key_layer_2 = value_layer_2 = None + attn_output_7 = attn_output_6.transpose(1, 2) + attn_output_6 = None + attn_output_8 = attn_output_7.reshape(1, 11, 768) + attn_output_7 = None + hidden_states_16 = torch._C._nn.linear( + attn_output_8, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_8 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_17 = torch.nn.functional.dropout( + hidden_states_16, 0.1, False, False + ) + hidden_states_16 = None + add_7 = hidden_states_17 + hidden_states_15 + hidden_states_17 = hidden_states_15 = None + hidden_states_18 = torch.nn.functional.layer_norm( + add_7, + (768,), + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_7 = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_19 = torch._C._nn.linear( + hidden_states_18, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_20 = torch._C._nn.gelu(hidden_states_19) + hidden_states_19 = None + hidden_states_21 = torch._C._nn.linear( + hidden_states_20, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_20 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_22 = torch.nn.functional.dropout( + hidden_states_21, 0.1, False, False + ) + hidden_states_21 = None + add_8 = hidden_states_22 + hidden_states_18 + hidden_states_22 = hidden_states_18 = None + hidden_states_23 = torch.nn.functional.layer_norm( + add_8, + (768,), + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_8 = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_2_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_18 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_9 = linear_18.view(1, -1, 12, 64) + linear_18 = None + query_layer_3 = view_9.transpose(1, 2) + view_9 = None + linear_19 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_10 = linear_19.view(1, -1, 12, 64) + linear_19 = None + key_layer_3 = view_10.transpose(1, 2) + view_10 = None + linear_20 = torch._C._nn.linear( + hidden_states_23, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_11 = linear_20.view(1, -1, 12, 64) + linear_20 = None + value_layer_3 = view_11.transpose(1, 2) + view_11 = None + attn_output_9 = torch._C._nn.scaled_dot_product_attention( + query_layer_3, + key_layer_3, + value_layer_3, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_3 = key_layer_3 = value_layer_3 = None + attn_output_10 = attn_output_9.transpose(1, 2) + attn_output_9 = None + attn_output_11 = attn_output_10.reshape(1, 11, 768) + attn_output_10 = None + hidden_states_24 = torch._C._nn.linear( + attn_output_11, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_11 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_25 = torch.nn.functional.dropout( + hidden_states_24, 0.1, False, False + ) + hidden_states_24 = None + add_9 = hidden_states_25 + hidden_states_23 + hidden_states_25 = hidden_states_23 = None + hidden_states_26 = torch.nn.functional.layer_norm( + add_9, + (768,), + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_9 = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_27 = torch._C._nn.linear( + hidden_states_26, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_28 = torch._C._nn.gelu(hidden_states_27) + hidden_states_27 = None + hidden_states_29 = torch._C._nn.linear( + hidden_states_28, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_28 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_30 = torch.nn.functional.dropout( + hidden_states_29, 0.1, False, False + ) + hidden_states_29 = None + add_10 = hidden_states_30 + hidden_states_26 + hidden_states_30 = hidden_states_26 = None + hidden_states_31 = torch.nn.functional.layer_norm( + add_10, + (768,), + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_10 = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_3_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_24 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_12 = linear_24.view(1, -1, 12, 64) + linear_24 = None + query_layer_4 = view_12.transpose(1, 2) + view_12 = None + linear_25 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_13 = linear_25.view(1, -1, 12, 64) + linear_25 = None + key_layer_4 = view_13.transpose(1, 2) + view_13 = None + linear_26 = torch._C._nn.linear( + hidden_states_31, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_14 = linear_26.view(1, -1, 12, 64) + linear_26 = None + value_layer_4 = view_14.transpose(1, 2) + view_14 = None + attn_output_12 = torch._C._nn.scaled_dot_product_attention( + query_layer_4, + key_layer_4, + value_layer_4, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_4 = key_layer_4 = value_layer_4 = None + attn_output_13 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_14 = attn_output_13.reshape(1, 11, 768) + attn_output_13 = None + hidden_states_32 = torch._C._nn.linear( + attn_output_14, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_14 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_33 = torch.nn.functional.dropout( + hidden_states_32, 0.1, False, False + ) + hidden_states_32 = None + add_11 = hidden_states_33 + hidden_states_31 + hidden_states_33 = hidden_states_31 = None + hidden_states_34 = torch.nn.functional.layer_norm( + add_11, + (768,), + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_11 = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_35 = torch._C._nn.linear( + hidden_states_34, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_36 = torch._C._nn.gelu(hidden_states_35) + hidden_states_35 = None + hidden_states_37 = torch._C._nn.linear( + hidden_states_36, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_36 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_38 = torch.nn.functional.dropout( + hidden_states_37, 0.1, False, False + ) + hidden_states_37 = None + add_12 = hidden_states_38 + hidden_states_34 + hidden_states_38 = hidden_states_34 = None + hidden_states_39 = torch.nn.functional.layer_norm( + add_12, + (768,), + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_12 = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_4_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_30 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_15 = linear_30.view(1, -1, 12, 64) + linear_30 = None + query_layer_5 = view_15.transpose(1, 2) + view_15 = None + linear_31 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_16 = linear_31.view(1, -1, 12, 64) + linear_31 = None + key_layer_5 = view_16.transpose(1, 2) + view_16 = None + linear_32 = torch._C._nn.linear( + hidden_states_39, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_17 = linear_32.view(1, -1, 12, 64) + linear_32 = None + value_layer_5 = view_17.transpose(1, 2) + view_17 = None + attn_output_15 = torch._C._nn.scaled_dot_product_attention( + query_layer_5, + key_layer_5, + value_layer_5, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_5 = key_layer_5 = value_layer_5 = None + attn_output_16 = attn_output_15.transpose(1, 2) + attn_output_15 = None + attn_output_17 = attn_output_16.reshape(1, 11, 768) + attn_output_16 = None + hidden_states_40 = torch._C._nn.linear( + attn_output_17, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_17 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_41 = torch.nn.functional.dropout( + hidden_states_40, 0.1, False, False + ) + hidden_states_40 = None + add_13 = hidden_states_41 + hidden_states_39 + hidden_states_41 = hidden_states_39 = None + hidden_states_42 = torch.nn.functional.layer_norm( + add_13, + (768,), + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_13 = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_43 = torch._C._nn.linear( + hidden_states_42, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_44 = torch._C._nn.gelu(hidden_states_43) + hidden_states_43 = None + hidden_states_45 = torch._C._nn.linear( + hidden_states_44, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_44 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_46 = torch.nn.functional.dropout( + hidden_states_45, 0.1, False, False + ) + hidden_states_45 = None + add_14 = hidden_states_46 + hidden_states_42 + hidden_states_46 = hidden_states_42 = None + hidden_states_47 = torch.nn.functional.layer_norm( + add_14, + (768,), + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_14 = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_5_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_36 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_18 = linear_36.view(1, -1, 12, 64) + linear_36 = None + query_layer_6 = view_18.transpose(1, 2) + view_18 = None + linear_37 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_19 = linear_37.view(1, -1, 12, 64) + linear_37 = None + key_layer_6 = view_19.transpose(1, 2) + view_19 = None + linear_38 = torch._C._nn.linear( + hidden_states_47, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_20 = linear_38.view(1, -1, 12, 64) + linear_38 = None + value_layer_6 = view_20.transpose(1, 2) + view_20 = None + attn_output_18 = torch._C._nn.scaled_dot_product_attention( + query_layer_6, + key_layer_6, + value_layer_6, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_6 = key_layer_6 = value_layer_6 = None + attn_output_19 = attn_output_18.transpose(1, 2) + attn_output_18 = None + attn_output_20 = attn_output_19.reshape(1, 11, 768) + attn_output_19 = None + hidden_states_48 = torch._C._nn.linear( + attn_output_20, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_20 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_49 = torch.nn.functional.dropout( + hidden_states_48, 0.1, False, False + ) + hidden_states_48 = None + add_15 = hidden_states_49 + hidden_states_47 + hidden_states_49 = hidden_states_47 = None + hidden_states_50 = torch.nn.functional.layer_norm( + add_15, + (768,), + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_15 = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_51 = torch._C._nn.linear( + hidden_states_50, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_52 = torch._C._nn.gelu(hidden_states_51) + hidden_states_51 = None + hidden_states_53 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_52 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_54 = torch.nn.functional.dropout( + hidden_states_53, 0.1, False, False + ) + hidden_states_53 = None + add_16 = hidden_states_54 + hidden_states_50 + hidden_states_54 = hidden_states_50 = None + hidden_states_55 = torch.nn.functional.layer_norm( + add_16, + (768,), + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_16 = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_6_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_42 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_21 = linear_42.view(1, -1, 12, 64) + linear_42 = None + query_layer_7 = view_21.transpose(1, 2) + view_21 = None + linear_43 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_22 = linear_43.view(1, -1, 12, 64) + linear_43 = None + key_layer_7 = view_22.transpose(1, 2) + view_22 = None + linear_44 = torch._C._nn.linear( + hidden_states_55, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_23 = linear_44.view(1, -1, 12, 64) + linear_44 = None + value_layer_7 = view_23.transpose(1, 2) + view_23 = None + attn_output_21 = torch._C._nn.scaled_dot_product_attention( + query_layer_7, + key_layer_7, + value_layer_7, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_7 = key_layer_7 = value_layer_7 = None + attn_output_22 = attn_output_21.transpose(1, 2) + attn_output_21 = None + attn_output_23 = attn_output_22.reshape(1, 11, 768) + attn_output_22 = None + hidden_states_56 = torch._C._nn.linear( + attn_output_23, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_23 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_57 = torch.nn.functional.dropout( + hidden_states_56, 0.1, False, False + ) + hidden_states_56 = None + add_17 = hidden_states_57 + hidden_states_55 + hidden_states_57 = hidden_states_55 = None + hidden_states_58 = torch.nn.functional.layer_norm( + add_17, + (768,), + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_17 = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_59 = torch._C._nn.linear( + hidden_states_58, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_60 = torch._C._nn.gelu(hidden_states_59) + hidden_states_59 = None + hidden_states_61 = torch._C._nn.linear( + hidden_states_60, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_60 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_62 = torch.nn.functional.dropout( + hidden_states_61, 0.1, False, False + ) + hidden_states_61 = None + add_18 = hidden_states_62 + hidden_states_58 + hidden_states_62 = hidden_states_58 = None + hidden_states_63 = torch.nn.functional.layer_norm( + add_18, + (768,), + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_18 = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_7_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_48 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_24 = linear_48.view(1, -1, 12, 64) + linear_48 = None + query_layer_8 = view_24.transpose(1, 2) + view_24 = None + linear_49 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_25 = linear_49.view(1, -1, 12, 64) + linear_49 = None + key_layer_8 = view_25.transpose(1, 2) + view_25 = None + linear_50 = torch._C._nn.linear( + hidden_states_63, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_26 = linear_50.view(1, -1, 12, 64) + linear_50 = None + value_layer_8 = view_26.transpose(1, 2) + view_26 = None + attn_output_24 = torch._C._nn.scaled_dot_product_attention( + query_layer_8, + key_layer_8, + value_layer_8, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_8 = key_layer_8 = value_layer_8 = None + attn_output_25 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_26 = attn_output_25.reshape(1, 11, 768) + attn_output_25 = None + hidden_states_64 = torch._C._nn.linear( + attn_output_26, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_26 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_65 = torch.nn.functional.dropout( + hidden_states_64, 0.1, False, False + ) + hidden_states_64 = None + add_19 = hidden_states_65 + hidden_states_63 + hidden_states_65 = hidden_states_63 = None + hidden_states_66 = torch.nn.functional.layer_norm( + add_19, + (768,), + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_19 = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_67 = torch._C._nn.linear( + hidden_states_66, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_68 = torch._C._nn.gelu(hidden_states_67) + hidden_states_67 = None + hidden_states_69 = torch._C._nn.linear( + hidden_states_68, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_68 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_70 = torch.nn.functional.dropout( + hidden_states_69, 0.1, False, False + ) + hidden_states_69 = None + add_20 = hidden_states_70 + hidden_states_66 + hidden_states_70 = hidden_states_66 = None + hidden_states_71 = torch.nn.functional.layer_norm( + add_20, + (768,), + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_20 = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_8_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_54 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_27 = linear_54.view(1, -1, 12, 64) + linear_54 = None + query_layer_9 = view_27.transpose(1, 2) + view_27 = None + linear_55 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_28 = linear_55.view(1, -1, 12, 64) + linear_55 = None + key_layer_9 = view_28.transpose(1, 2) + view_28 = None + linear_56 = torch._C._nn.linear( + hidden_states_71, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_29 = linear_56.view(1, -1, 12, 64) + linear_56 = None + value_layer_9 = view_29.transpose(1, 2) + view_29 = None + attn_output_27 = torch._C._nn.scaled_dot_product_attention( + query_layer_9, + key_layer_9, + value_layer_9, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_9 = key_layer_9 = value_layer_9 = None + attn_output_28 = attn_output_27.transpose(1, 2) + attn_output_27 = None + attn_output_29 = attn_output_28.reshape(1, 11, 768) + attn_output_28 = None + hidden_states_72 = torch._C._nn.linear( + attn_output_29, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_29 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_73 = torch.nn.functional.dropout( + hidden_states_72, 0.1, False, False + ) + hidden_states_72 = None + add_21 = hidden_states_73 + hidden_states_71 + hidden_states_73 = hidden_states_71 = None + hidden_states_74 = torch.nn.functional.layer_norm( + add_21, + (768,), + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_21 = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_75 = torch._C._nn.linear( + hidden_states_74, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_76 = torch._C._nn.gelu(hidden_states_75) + hidden_states_75 = None + hidden_states_77 = torch._C._nn.linear( + hidden_states_76, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_76 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_78 = torch.nn.functional.dropout( + hidden_states_77, 0.1, False, False + ) + hidden_states_77 = None + add_22 = hidden_states_78 + hidden_states_74 + hidden_states_78 = hidden_states_74 = None + hidden_states_79 = torch.nn.functional.layer_norm( + add_22, + (768,), + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_22 = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_9_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_60 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_30 = linear_60.view(1, -1, 12, 64) + linear_60 = None + query_layer_10 = view_30.transpose(1, 2) + view_30 = None + linear_61 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_31 = linear_61.view(1, -1, 12, 64) + linear_61 = None + key_layer_10 = view_31.transpose(1, 2) + view_31 = None + linear_62 = torch._C._nn.linear( + hidden_states_79, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_32 = linear_62.view(1, -1, 12, 64) + linear_62 = None + value_layer_10 = view_32.transpose(1, 2) + view_32 = None + attn_output_30 = torch._C._nn.scaled_dot_product_attention( + query_layer_10, + key_layer_10, + value_layer_10, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_10 = key_layer_10 = value_layer_10 = None + attn_output_31 = attn_output_30.transpose(1, 2) + attn_output_30 = None + attn_output_32 = attn_output_31.reshape(1, 11, 768) + attn_output_31 = None + hidden_states_80 = torch._C._nn.linear( + attn_output_32, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_32 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_81 = torch.nn.functional.dropout( + hidden_states_80, 0.1, False, False + ) + hidden_states_80 = None + add_23 = hidden_states_81 + hidden_states_79 + hidden_states_81 = hidden_states_79 = None + hidden_states_82 = torch.nn.functional.layer_norm( + add_23, + (768,), + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_23 = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_83 = torch._C._nn.linear( + hidden_states_82, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_84 = torch._C._nn.gelu(hidden_states_83) + hidden_states_83 = None + hidden_states_85 = torch._C._nn.linear( + hidden_states_84, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_84 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_86 = torch.nn.functional.dropout( + hidden_states_85, 0.1, False, False + ) + hidden_states_85 = None + add_24 = hidden_states_86 + hidden_states_82 + hidden_states_86 = hidden_states_82 = None + hidden_states_87 = torch.nn.functional.layer_norm( + add_24, + (768,), + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_24 = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_10_modules_output_modules_layer_norm_parameters_bias_ = (None) + linear_66 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_ = (None) + view_33 = linear_66.view(1, -1, 12, 64) + linear_66 = None + query_layer_11 = view_33.transpose(1, 2) + view_33 = None + linear_67 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_ = (None) + view_34 = linear_67.view(1, -1, 12, 64) + linear_67 = None + key_layer_11 = view_34.transpose(1, 2) + view_34 = None + linear_68 = torch._C._nn.linear( + hidden_states_87, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_ = (None) + view_35 = linear_68.view(1, -1, 12, 64) + linear_68 = None + value_layer_11 = view_35.transpose(1, 2) + view_35 = None + attn_output_33 = torch._C._nn.scaled_dot_product_attention( + query_layer_11, + key_layer_11, + value_layer_11, + attn_mask=extended_attention_mask, + dropout_p=0.0, + is_causal=False, + ) + query_layer_11 = key_layer_11 = value_layer_11 = extended_attention_mask = None + attn_output_34 = attn_output_33.transpose(1, 2) + attn_output_33 = None + attn_output_35 = attn_output_34.reshape(1, 11, 768) + attn_output_34 = None + hidden_states_88 = torch._C._nn.linear( + attn_output_35, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_, + ) + attn_output_35 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_89 = torch.nn.functional.dropout( + hidden_states_88, 0.1, False, False + ) + hidden_states_88 = None + add_25 = hidden_states_89 + hidden_states_87 + hidden_states_89 = hidden_states_87 = None + hidden_states_90 = torch.nn.functional.layer_norm( + add_25, + (768,), + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_25 = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_layer_norm_parameters_bias_ = (None) + hidden_states_91 = torch._C._nn.linear( + hidden_states_90, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_, + ) + l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_ = (None) + hidden_states_92 = torch._C._nn.gelu(hidden_states_91) + hidden_states_91 = None + hidden_states_93 = torch._C._nn.linear( + hidden_states_92, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_, + ) + hidden_states_92 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_ = (None) + hidden_states_94 = torch.nn.functional.dropout( + hidden_states_93, 0.1, False, False + ) + hidden_states_93 = None + add_26 = hidden_states_94 + hidden_states_90 + hidden_states_94 = hidden_states_90 = None + hidden_states_95 = torch.nn.functional.layer_norm( + add_26, + (768,), + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_, + l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_, + 1e-05, + ) + add_26 = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_weight_ = l_self_modules_encoder_modules_layer_modules_11_modules_output_modules_layer_norm_parameters_bias_ = (None) + first_token_tensor = hidden_states_95[(slice(None, None, None), 0)] + pooled_output = torch._C._nn.linear( + first_token_tensor, + l_self_modules_pooler_modules_dense_parameters_weight_, + l_self_modules_pooler_modules_dense_parameters_bias_, + ) + first_token_tensor = ( + l_self_modules_pooler_modules_dense_parameters_weight_ + ) = l_self_modules_pooler_modules_dense_parameters_bias_ = None + pooled_output_1 = torch.tanh(pooled_output) + pooled_output = None + return (hidden_states_95, pooled_output_1) diff --git a/samples/transformers-auto-model/roberta-base/weight_meta.py b/samples/transformers-auto-model/roberta-base/weight_meta.py new file mode 100644 index 0000000000..5e02bbbb46 --- /dev/null +++ b/samples/transformers-auto-model/roberta-base/weight_meta.py @@ -0,0 +1,2533 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 45288, 15721, 16, 341, 7, 14660, 38163, 36386, 4, 2] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_buffers_token_type_ids_: + name = "L_self_modules_embeddings_buffers_token_type_ids_" + shape = [1, 514] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + ] + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_word_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_word_embeddings_parameters_weight_" + shape = [50265, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.131 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_token_type_embeddings_parameters_weight_" + shape = [1, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.000 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_position_embeddings_parameters_weight_: + name = "L_self_modules_embeddings_modules_position_embeddings_parameters_weight_" + shape = [514, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.352 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_embeddings_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_embeddings_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 11] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.102 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.016 + std = 0.338 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.101 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.001 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.606 + std = 0.079 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.106 + std = 0.164 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.100 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.565 + std = 0.104 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_0_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.030 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.243 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.002 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.666 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.122 + std = 0.095 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.059 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.091 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.629 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_1_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.037 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.020 + std = 0.167 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.765 + std = 0.121 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.093 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.092 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.089 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.665 + std = 0.112 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_2_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.036 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.163 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.003 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.791 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.082 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.084 + std = 0.081 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.678 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_3_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.035 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.073 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.135 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.002 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.816 + std = 0.118 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.078 + std = 0.063 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.082 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.728 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_4_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.033 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.127 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.005 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.051 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.827 + std = 0.113 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.065 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.071 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.726 + std = 0.105 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_5_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.025 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.008 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.806 + std = 0.111 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.069 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.076 + std = 0.091 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.731 + std = 0.113 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_6_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.028 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.087 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.017 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.026 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.828 + std = 0.108 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.062 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.071 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.766 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_7_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.031 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.022 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.843 + std = 0.100 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.076 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.070 + std = 0.093 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.798 + std = 0.086 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_8_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.036 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.066 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.123 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.065 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.083 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.857 + std = 0.088 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.081 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.067 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.802 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_9_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.038 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.152 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.018 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.023 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.807 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.069 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.060 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.791 + std = 0.064 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_10_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.032 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_query_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.143 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.067 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_key_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.047 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_self_modules_value_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.697 + std = 0.061 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_attention_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.083 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_weight_" + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.008 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_intermediate_modules_dense_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.054 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_weight_" + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.758 + std = 0.102 + data = None + + +class Program_weight_tensor_meta_L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_: + name = "L_self_modules_encoder_modules_layer_modules_11_modules_output_modules_LayerNorm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.040 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_weight_: + name = "L_self_modules_pooler_modules_dense_parameters_weight_" + shape = [768, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.020 + data = None + + +class Program_weight_tensor_meta_L_self_modules_pooler_modules_dense_parameters_bias_: + name = "L_self_modules_pooler_modules_dense_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.000 + data = None diff --git a/samples/transformers-auto-model/t5-small/graph_hash.txt b/samples/transformers-auto-model/t5-small/graph_hash.txt new file mode 100644 index 0000000000..4b440b15f4 --- /dev/null +++ b/samples/transformers-auto-model/t5-small/graph_hash.txt @@ -0,0 +1 @@ +140e22f0f6b1b0aa7a9438b9253000ee6064c918c0a1df9888a3552ed3c77900 \ No newline at end of file diff --git a/samples/transformers-auto-model/t5-small/graph_net.json b/samples/transformers-auto-model/t5-small/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/t5-small/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/t5-small/input_meta.py b/samples/transformers-auto-model/t5-small/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/t5-small/input_tensor_constraints.py b/samples/transformers-auto-model/t5-small/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/t5-small/model.py b/samples/transformers-auto-model/t5-small/model.py new file mode 100644 index 0000000000..e039fdcee2 --- /dev/null +++ b/samples/transformers-auto-model/t5-small/model.py @@ -0,0 +1,1623 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_inputs_embeds_: torch.Tensor, + L_encoder_hidden_states_: torch.Tensor, + L_encoder_attention_mask_: torch.Tensor, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_final_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + ): + l_inputs_embeds_ = L_inputs_embeds_ + l_encoder_hidden_states_ = L_encoder_hidden_states_ + l_encoder_attention_mask_ = L_encoder_attention_mask_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_relative_attention_bias_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_ + l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_ + l_self_modules_final_layer_norm_parameters_weight_ = ( + L_self_modules_final_layer_norm_parameters_weight_ + ) + cache_position = torch.arange(0, 12, device=device(type="cuda", index=0)) + causal_mask = torch.full( + (12, 13), + fill_value=-3.4028234663852886e38, + dtype=torch.float32, + device=device(type="cuda", index=0), + ) + causal_mask_1 = torch.triu(causal_mask, diagonal=1) + causal_mask = None + arange_1 = torch.arange(13, device=device(type="cuda", index=0)) + reshape = cache_position.reshape(-1, 1) + gt = arange_1 > reshape + arange_1 = reshape = None + causal_mask_1 *= gt + causal_mask_2 = causal_mask_1 + causal_mask_1 = gt = None + getitem = causal_mask_2[ + (None, None, slice(None, None, None), slice(None, None, None)) + ] + causal_mask_2 = None + causal_mask_3 = getitem.expand(1, 1, -1, -1) + getitem = None + encoder_extended_attention_mask = l_encoder_attention_mask_[ + (slice(None, None, None), None, None, slice(None, None, None)) + ] + l_encoder_attention_mask_ = None + encoder_extended_attention_mask_1 = encoder_extended_attention_mask.to( + dtype=torch.float32 + ) + encoder_extended_attention_mask = None + sub = 1.0 - encoder_extended_attention_mask_1 + encoder_extended_attention_mask_1 = None + encoder_extended_attention_mask_2 = sub * -3.4028234663852886e38 + sub = None + hidden_states = torch.nn.functional.dropout(l_inputs_embeds_, 0.1, False, False) + l_inputs_embeds_ = None + to_1 = hidden_states.to(torch.float32) + pow_1 = to_1.pow(2) + to_1 = None + variance = pow_1.mean(-1, keepdim=True) + pow_1 = None + add = variance + 1e-06 + variance = None + rsqrt = torch.rsqrt(add) + add = None + hidden_states_1 = hidden_states * rsqrt + rsqrt = None + normed_hidden_states = ( + l_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_1 + ) + l_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_1 + ) = None + query_states = torch._C._nn.linear( + normed_hidden_states, + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view = query_states.view(1, -1, 8, 64) + query_states = None + query_states_1 = view.transpose(1, 2) + view = None + key_states = torch._C._nn.linear( + normed_hidden_states, + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states = torch._C._nn.linear( + normed_hidden_states, + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states = l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_1 = key_states.view(1, -1, 8, 64) + key_states = None + key_states_1 = view_1.transpose(1, 2) + view_1 = None + view_2 = value_states.view(1, -1, 8, 64) + value_states = None + value_states_1 = view_2.transpose(1, 2) + view_2 = None + transpose_3 = key_states_1.transpose(3, 2) + scores = torch.matmul(query_states_1, transpose_3) + query_states_1 = transpose_3 = None + getitem_2 = cache_position[-1] + real_seq_length = getitem_2 + 1 + getitem_2 = real_seq_length = None + getitem_3 = cache_position[(slice(None, None, None), None)] + context_position = getitem_3.to(device(type="cuda", index=0)) + getitem_3 = None + arange_2 = torch.arange( + 12, dtype=torch.int64, device=device(type="cuda", index=0) + ) + memory_position = arange_2[(None, slice(None, None, None))] + arange_2 = None + relative_position = memory_position - context_position + memory_position = context_position = None + zeros_like = torch.zeros_like(relative_position) + min_1 = torch.min(relative_position, zeros_like) + relative_position = zeros_like = None + relative_position_1 = -min_1 + min_1 = None + is_small = relative_position_1 < 16 + float_1 = relative_position_1.float() + truediv = float_1 / 16 + float_1 = None + log = torch.log(truediv) + truediv = None + truediv_1 = log / 2.0794415416798357 + log = None + mul_3 = truediv_1 * 16 + truediv_1 = None + to_3 = mul_3.to(torch.int64) + mul_3 = None + relative_position_if_large = 16 + to_3 + to_3 = None + full_like = torch.full_like(relative_position_if_large, 31) + relative_position_if_large_1 = torch.min(relative_position_if_large, full_like) + relative_position_if_large = full_like = None + where = torch.where(is_small, relative_position_1, relative_position_if_large_1) + is_small = relative_position_1 = relative_position_if_large_1 = None + relative_buckets = 0 + where + where = None + values = torch.nn.functional.embedding( + relative_buckets, + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_relative_attention_bias_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + relative_buckets = l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_relative_attention_bias_parameters_weight_ = (None) + permute = values.permute([2, 0, 1]) + values = None + values_1 = permute.unsqueeze(0) + permute = None + position_bias = values_1[ + ( + slice(None, None, None), + slice(None, None, None), + slice(-12, None, None), + slice(None, None, None), + ) + ] + values_1 = None + causal_mask_4 = causal_mask_3[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 12, None), + ) + ] + causal_mask_3 = None + position_bias_1 = position_bias + causal_mask_4 + position_bias = causal_mask_4 = None + scores += position_bias_1 + scores_1 = scores + scores = None + float_2 = scores_1.float() + softmax = torch.nn.functional.softmax(float_2, dim=-1) + float_2 = None + attn_weights = softmax.type_as(scores_1) + softmax = scores_1 = None + attn_weights_1 = torch.nn.functional.dropout( + attn_weights, p=0.1, training=False + ) + attn_weights = None + attn_output = torch.matmul(attn_weights_1, value_states_1) + attn_weights_1 = None + transpose_4 = attn_output.transpose(1, 2) + attn_output = None + attn_output_1 = transpose_4.contiguous() + transpose_4 = None + attn_output_2 = attn_output_1.view(1, -1, 512) + attn_output_1 = None + attn_output_3 = torch._C._nn.linear( + attn_output_2, + l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_2 = l_self_modules_block_modules_0_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_2 = torch.nn.functional.dropout(attn_output_3, 0.1, False, False) + attn_output_3 = None + hidden_states_2 = hidden_states + dropout_2 + hidden_states = dropout_2 = None + getitem_7 = cache_position[-1] + real_seq_length_1 = getitem_7 + 1 + getitem_7 = real_seq_length_1 = None + to_4 = hidden_states_2.to(torch.float32) + pow_2 = to_4.pow(2) + to_4 = None + variance_1 = pow_2.mean(-1, keepdim=True) + pow_2 = None + add_7 = variance_1 + 1e-06 + variance_1 = None + rsqrt_1 = torch.rsqrt(add_7) + add_7 = None + hidden_states_3 = hidden_states_2 * rsqrt_1 + rsqrt_1 = None + normed_hidden_states_1 = ( + l_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_3 + ) + l_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_3 + ) = None + query_states_2 = torch._C._nn.linear( + normed_hidden_states_1, + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_1 = l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_4 = query_states_2.view(1, -1, 8, 64) + query_states_2 = None + query_states_3 = view_4.transpose(1, 2) + view_4 = None + key_states_2 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_2 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = ( + None + ) + view_5 = key_states_2.view(1, -1, 8, 64) + key_states_2 = None + key_states_3 = view_5.transpose(1, 2) + view_5 = None + view_6 = value_states_2.view(1, -1, 8, 64) + value_states_2 = None + value_states_3 = view_6.transpose(1, 2) + view_6 = None + transpose_8 = key_states_3.transpose(3, 2) + scores_2 = torch.matmul(query_states_3, transpose_8) + query_states_3 = transpose_8 = None + position_bias_2 = torch.zeros( + (1, 8, 12, 12), device=device(type="cuda", index=0), dtype=torch.float32 + ) + causal_mask_5 = encoder_extended_attention_mask_2[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + slice(None, 12, None), + ) + ] + encoder_extended_attention_mask_2 = None + position_bias_3 = position_bias_2 + causal_mask_5 + position_bias_2 = causal_mask_5 = None + scores_2 += position_bias_3 + scores_3 = scores_2 + scores_2 = None + float_3 = scores_3.float() + softmax_1 = torch.nn.functional.softmax(float_3, dim=-1) + float_3 = None + attn_weights_2 = softmax_1.type_as(scores_3) + softmax_1 = scores_3 = None + attn_weights_3 = torch.nn.functional.dropout( + attn_weights_2, p=0.1, training=False + ) + attn_weights_2 = None + attn_output_4 = torch.matmul(attn_weights_3, value_states_3) + attn_weights_3 = None + transpose_9 = attn_output_4.transpose(1, 2) + attn_output_4 = None + attn_output_5 = transpose_9.contiguous() + transpose_9 = None + attn_output_6 = attn_output_5.view(1, -1, 512) + attn_output_5 = None + attn_output_7 = torch._C._nn.linear( + attn_output_6, + l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_6 = l_self_modules_block_modules_0_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_4 = torch.nn.functional.dropout(attn_output_7, 0.1, False, False) + attn_output_7 = None + layer_output = hidden_states_2 + dropout_4 + hidden_states_2 = dropout_4 = None + to_5 = layer_output.to(torch.float32) + pow_3 = to_5.pow(2) + to_5 = None + variance_2 = pow_3.mean(-1, keepdim=True) + pow_3 = None + add_10 = variance_2 + 1e-06 + variance_2 = None + rsqrt_2 = torch.rsqrt(add_10) + add_10 = None + hidden_states_4 = layer_output * rsqrt_2 + rsqrt_2 = None + forwarded_states = ( + l_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_4 + ) + l_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_4 + ) = None + hidden_states_5 = torch._C._nn.linear( + forwarded_states, + l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states = l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_6 = torch.nn.functional.relu(hidden_states_5, inplace=False) + hidden_states_5 = None + hidden_states_7 = torch.nn.functional.dropout( + hidden_states_6, 0.1, False, False + ) + hidden_states_6 = None + hidden_states_8 = torch._C._nn.linear( + hidden_states_7, + l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_7 = l_self_modules_block_modules_0_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_6 = torch.nn.functional.dropout(hidden_states_8, 0.1, False, False) + hidden_states_8 = None + hidden_states_9 = layer_output + dropout_6 + layer_output = dropout_6 = None + to_6 = hidden_states_9.to(torch.float32) + pow_4 = to_6.pow(2) + to_6 = None + variance_3 = pow_4.mean(-1, keepdim=True) + pow_4 = None + add_12 = variance_3 + 1e-06 + variance_3 = None + rsqrt_3 = torch.rsqrt(add_12) + add_12 = None + hidden_states_10 = hidden_states_9 * rsqrt_3 + rsqrt_3 = None + normed_hidden_states_2 = ( + l_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_10 + ) + l_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_10 + ) = None + query_states_4 = torch._C._nn.linear( + normed_hidden_states_2, + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view_8 = query_states_4.view(1, -1, 8, 64) + query_states_4 = None + query_states_5 = view_8.transpose(1, 2) + view_8 = None + key_states_4 = torch._C._nn.linear( + normed_hidden_states_2, + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_4 = torch._C._nn.linear( + normed_hidden_states_2, + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states_2 = l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_9 = key_states_4.view(1, -1, 8, 64) + key_states_4 = None + key_states_5 = view_9.transpose(1, 2) + view_9 = None + view_10 = value_states_4.view(1, -1, 8, 64) + value_states_4 = None + value_states_5 = view_10.transpose(1, 2) + view_10 = None + transpose_13 = key_states_5.transpose(3, 2) + scores_4 = torch.matmul(query_states_5, transpose_13) + query_states_5 = transpose_13 = None + scores_4 += position_bias_1 + scores_5 = scores_4 + scores_4 = None + float_4 = scores_5.float() + softmax_2 = torch.nn.functional.softmax(float_4, dim=-1) + float_4 = None + attn_weights_4 = softmax_2.type_as(scores_5) + softmax_2 = scores_5 = None + attn_weights_5 = torch.nn.functional.dropout( + attn_weights_4, p=0.1, training=False + ) + attn_weights_4 = None + attn_output_8 = torch.matmul(attn_weights_5, value_states_5) + attn_weights_5 = None + transpose_14 = attn_output_8.transpose(1, 2) + attn_output_8 = None + attn_output_9 = transpose_14.contiguous() + transpose_14 = None + attn_output_10 = attn_output_9.view(1, -1, 512) + attn_output_9 = None + attn_output_11 = torch._C._nn.linear( + attn_output_10, + l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_10 = l_self_modules_block_modules_1_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_8 = torch.nn.functional.dropout(attn_output_11, 0.1, False, False) + attn_output_11 = None + hidden_states_11 = hidden_states_9 + dropout_8 + hidden_states_9 = dropout_8 = None + getitem_9 = cache_position[-1] + add_14 = getitem_9 + 1 + getitem_9 = add_14 = None + to_7 = hidden_states_11.to(torch.float32) + pow_5 = to_7.pow(2) + to_7 = None + variance_4 = pow_5.mean(-1, keepdim=True) + pow_5 = None + add_15 = variance_4 + 1e-06 + variance_4 = None + rsqrt_4 = torch.rsqrt(add_15) + add_15 = None + hidden_states_12 = hidden_states_11 * rsqrt_4 + rsqrt_4 = None + normed_hidden_states_3 = ( + l_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_12 + ) + l_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_12 + ) = None + query_states_6 = torch._C._nn.linear( + normed_hidden_states_3, + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_3 = l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_12 = query_states_6.view(1, -1, 8, 64) + query_states_6 = None + query_states_7 = view_12.transpose(1, 2) + view_12 = None + key_states_6 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_6 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = ( + None + ) + view_13 = key_states_6.view(1, -1, 8, 64) + key_states_6 = None + key_states_7 = view_13.transpose(1, 2) + view_13 = None + view_14 = value_states_6.view(1, -1, 8, 64) + value_states_6 = None + value_states_7 = view_14.transpose(1, 2) + view_14 = None + transpose_18 = key_states_7.transpose(3, 2) + scores_6 = torch.matmul(query_states_7, transpose_18) + query_states_7 = transpose_18 = None + scores_6 += position_bias_3 + scores_7 = scores_6 + scores_6 = None + float_5 = scores_7.float() + softmax_3 = torch.nn.functional.softmax(float_5, dim=-1) + float_5 = None + attn_weights_6 = softmax_3.type_as(scores_7) + softmax_3 = scores_7 = None + attn_weights_7 = torch.nn.functional.dropout( + attn_weights_6, p=0.1, training=False + ) + attn_weights_6 = None + attn_output_12 = torch.matmul(attn_weights_7, value_states_7) + attn_weights_7 = None + transpose_19 = attn_output_12.transpose(1, 2) + attn_output_12 = None + attn_output_13 = transpose_19.contiguous() + transpose_19 = None + attn_output_14 = attn_output_13.view(1, -1, 512) + attn_output_13 = None + attn_output_15 = torch._C._nn.linear( + attn_output_14, + l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_14 = l_self_modules_block_modules_1_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_10 = torch.nn.functional.dropout(attn_output_15, 0.1, False, False) + attn_output_15 = None + layer_output_1 = hidden_states_11 + dropout_10 + hidden_states_11 = dropout_10 = None + to_8 = layer_output_1.to(torch.float32) + pow_6 = to_8.pow(2) + to_8 = None + variance_5 = pow_6.mean(-1, keepdim=True) + pow_6 = None + add_17 = variance_5 + 1e-06 + variance_5 = None + rsqrt_5 = torch.rsqrt(add_17) + add_17 = None + hidden_states_13 = layer_output_1 * rsqrt_5 + rsqrt_5 = None + forwarded_states_1 = ( + l_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_13 + ) + l_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_13 + ) = None + hidden_states_14 = torch._C._nn.linear( + forwarded_states_1, + l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states_1 = l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_15 = torch.nn.functional.relu(hidden_states_14, inplace=False) + hidden_states_14 = None + hidden_states_16 = torch.nn.functional.dropout( + hidden_states_15, 0.1, False, False + ) + hidden_states_15 = None + hidden_states_17 = torch._C._nn.linear( + hidden_states_16, + l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_16 = l_self_modules_block_modules_1_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_12 = torch.nn.functional.dropout(hidden_states_17, 0.1, False, False) + hidden_states_17 = None + hidden_states_18 = layer_output_1 + dropout_12 + layer_output_1 = dropout_12 = None + to_9 = hidden_states_18.to(torch.float32) + pow_7 = to_9.pow(2) + to_9 = None + variance_6 = pow_7.mean(-1, keepdim=True) + pow_7 = None + add_19 = variance_6 + 1e-06 + variance_6 = None + rsqrt_6 = torch.rsqrt(add_19) + add_19 = None + hidden_states_19 = hidden_states_18 * rsqrt_6 + rsqrt_6 = None + normed_hidden_states_4 = ( + l_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_19 + ) + l_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_19 + ) = None + query_states_8 = torch._C._nn.linear( + normed_hidden_states_4, + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view_16 = query_states_8.view(1, -1, 8, 64) + query_states_8 = None + query_states_9 = view_16.transpose(1, 2) + view_16 = None + key_states_8 = torch._C._nn.linear( + normed_hidden_states_4, + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_8 = torch._C._nn.linear( + normed_hidden_states_4, + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states_4 = l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_17 = key_states_8.view(1, -1, 8, 64) + key_states_8 = None + key_states_9 = view_17.transpose(1, 2) + view_17 = None + view_18 = value_states_8.view(1, -1, 8, 64) + value_states_8 = None + value_states_9 = view_18.transpose(1, 2) + view_18 = None + transpose_23 = key_states_9.transpose(3, 2) + scores_8 = torch.matmul(query_states_9, transpose_23) + query_states_9 = transpose_23 = None + scores_8 += position_bias_1 + scores_9 = scores_8 + scores_8 = None + float_6 = scores_9.float() + softmax_4 = torch.nn.functional.softmax(float_6, dim=-1) + float_6 = None + attn_weights_8 = softmax_4.type_as(scores_9) + softmax_4 = scores_9 = None + attn_weights_9 = torch.nn.functional.dropout( + attn_weights_8, p=0.1, training=False + ) + attn_weights_8 = None + attn_output_16 = torch.matmul(attn_weights_9, value_states_9) + attn_weights_9 = None + transpose_24 = attn_output_16.transpose(1, 2) + attn_output_16 = None + attn_output_17 = transpose_24.contiguous() + transpose_24 = None + attn_output_18 = attn_output_17.view(1, -1, 512) + attn_output_17 = None + attn_output_19 = torch._C._nn.linear( + attn_output_18, + l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_18 = l_self_modules_block_modules_2_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_14 = torch.nn.functional.dropout(attn_output_19, 0.1, False, False) + attn_output_19 = None + hidden_states_20 = hidden_states_18 + dropout_14 + hidden_states_18 = dropout_14 = None + getitem_10 = cache_position[-1] + add_21 = getitem_10 + 1 + getitem_10 = add_21 = None + to_10 = hidden_states_20.to(torch.float32) + pow_8 = to_10.pow(2) + to_10 = None + variance_7 = pow_8.mean(-1, keepdim=True) + pow_8 = None + add_22 = variance_7 + 1e-06 + variance_7 = None + rsqrt_7 = torch.rsqrt(add_22) + add_22 = None + hidden_states_21 = hidden_states_20 * rsqrt_7 + rsqrt_7 = None + normed_hidden_states_5 = ( + l_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_21 + ) + l_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_21 + ) = None + query_states_10 = torch._C._nn.linear( + normed_hidden_states_5, + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_5 = l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_20 = query_states_10.view(1, -1, 8, 64) + query_states_10 = None + query_states_11 = view_20.transpose(1, 2) + view_20 = None + key_states_10 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_10 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = ( + None + ) + view_21 = key_states_10.view(1, -1, 8, 64) + key_states_10 = None + key_states_11 = view_21.transpose(1, 2) + view_21 = None + view_22 = value_states_10.view(1, -1, 8, 64) + value_states_10 = None + value_states_11 = view_22.transpose(1, 2) + view_22 = None + transpose_28 = key_states_11.transpose(3, 2) + scores_10 = torch.matmul(query_states_11, transpose_28) + query_states_11 = transpose_28 = None + scores_10 += position_bias_3 + scores_11 = scores_10 + scores_10 = None + float_7 = scores_11.float() + softmax_5 = torch.nn.functional.softmax(float_7, dim=-1) + float_7 = None + attn_weights_10 = softmax_5.type_as(scores_11) + softmax_5 = scores_11 = None + attn_weights_11 = torch.nn.functional.dropout( + attn_weights_10, p=0.1, training=False + ) + attn_weights_10 = None + attn_output_20 = torch.matmul(attn_weights_11, value_states_11) + attn_weights_11 = None + transpose_29 = attn_output_20.transpose(1, 2) + attn_output_20 = None + attn_output_21 = transpose_29.contiguous() + transpose_29 = None + attn_output_22 = attn_output_21.view(1, -1, 512) + attn_output_21 = None + attn_output_23 = torch._C._nn.linear( + attn_output_22, + l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_22 = l_self_modules_block_modules_2_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_16 = torch.nn.functional.dropout(attn_output_23, 0.1, False, False) + attn_output_23 = None + layer_output_2 = hidden_states_20 + dropout_16 + hidden_states_20 = dropout_16 = None + to_11 = layer_output_2.to(torch.float32) + pow_9 = to_11.pow(2) + to_11 = None + variance_8 = pow_9.mean(-1, keepdim=True) + pow_9 = None + add_24 = variance_8 + 1e-06 + variance_8 = None + rsqrt_8 = torch.rsqrt(add_24) + add_24 = None + hidden_states_22 = layer_output_2 * rsqrt_8 + rsqrt_8 = None + forwarded_states_2 = ( + l_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_22 + ) + l_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_22 + ) = None + hidden_states_23 = torch._C._nn.linear( + forwarded_states_2, + l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states_2 = l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_24 = torch.nn.functional.relu(hidden_states_23, inplace=False) + hidden_states_23 = None + hidden_states_25 = torch.nn.functional.dropout( + hidden_states_24, 0.1, False, False + ) + hidden_states_24 = None + hidden_states_26 = torch._C._nn.linear( + hidden_states_25, + l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_25 = l_self_modules_block_modules_2_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_18 = torch.nn.functional.dropout(hidden_states_26, 0.1, False, False) + hidden_states_26 = None + hidden_states_27 = layer_output_2 + dropout_18 + layer_output_2 = dropout_18 = None + to_12 = hidden_states_27.to(torch.float32) + pow_10 = to_12.pow(2) + to_12 = None + variance_9 = pow_10.mean(-1, keepdim=True) + pow_10 = None + add_26 = variance_9 + 1e-06 + variance_9 = None + rsqrt_9 = torch.rsqrt(add_26) + add_26 = None + hidden_states_28 = hidden_states_27 * rsqrt_9 + rsqrt_9 = None + normed_hidden_states_6 = ( + l_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_28 + ) + l_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_28 + ) = None + query_states_12 = torch._C._nn.linear( + normed_hidden_states_6, + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view_24 = query_states_12.view(1, -1, 8, 64) + query_states_12 = None + query_states_13 = view_24.transpose(1, 2) + view_24 = None + key_states_12 = torch._C._nn.linear( + normed_hidden_states_6, + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_12 = torch._C._nn.linear( + normed_hidden_states_6, + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states_6 = l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_25 = key_states_12.view(1, -1, 8, 64) + key_states_12 = None + key_states_13 = view_25.transpose(1, 2) + view_25 = None + view_26 = value_states_12.view(1, -1, 8, 64) + value_states_12 = None + value_states_13 = view_26.transpose(1, 2) + view_26 = None + transpose_33 = key_states_13.transpose(3, 2) + scores_12 = torch.matmul(query_states_13, transpose_33) + query_states_13 = transpose_33 = None + scores_12 += position_bias_1 + scores_13 = scores_12 + scores_12 = None + float_8 = scores_13.float() + softmax_6 = torch.nn.functional.softmax(float_8, dim=-1) + float_8 = None + attn_weights_12 = softmax_6.type_as(scores_13) + softmax_6 = scores_13 = None + attn_weights_13 = torch.nn.functional.dropout( + attn_weights_12, p=0.1, training=False + ) + attn_weights_12 = None + attn_output_24 = torch.matmul(attn_weights_13, value_states_13) + attn_weights_13 = None + transpose_34 = attn_output_24.transpose(1, 2) + attn_output_24 = None + attn_output_25 = transpose_34.contiguous() + transpose_34 = None + attn_output_26 = attn_output_25.view(1, -1, 512) + attn_output_25 = None + attn_output_27 = torch._C._nn.linear( + attn_output_26, + l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_26 = l_self_modules_block_modules_3_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_20 = torch.nn.functional.dropout(attn_output_27, 0.1, False, False) + attn_output_27 = None + hidden_states_29 = hidden_states_27 + dropout_20 + hidden_states_27 = dropout_20 = None + getitem_11 = cache_position[-1] + add_28 = getitem_11 + 1 + getitem_11 = add_28 = None + to_13 = hidden_states_29.to(torch.float32) + pow_11 = to_13.pow(2) + to_13 = None + variance_10 = pow_11.mean(-1, keepdim=True) + pow_11 = None + add_29 = variance_10 + 1e-06 + variance_10 = None + rsqrt_10 = torch.rsqrt(add_29) + add_29 = None + hidden_states_30 = hidden_states_29 * rsqrt_10 + rsqrt_10 = None + normed_hidden_states_7 = ( + l_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_30 + ) + l_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_30 + ) = None + query_states_14 = torch._C._nn.linear( + normed_hidden_states_7, + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_7 = l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_28 = query_states_14.view(1, -1, 8, 64) + query_states_14 = None + query_states_15 = view_28.transpose(1, 2) + view_28 = None + key_states_14 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_14 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = ( + None + ) + view_29 = key_states_14.view(1, -1, 8, 64) + key_states_14 = None + key_states_15 = view_29.transpose(1, 2) + view_29 = None + view_30 = value_states_14.view(1, -1, 8, 64) + value_states_14 = None + value_states_15 = view_30.transpose(1, 2) + view_30 = None + transpose_38 = key_states_15.transpose(3, 2) + scores_14 = torch.matmul(query_states_15, transpose_38) + query_states_15 = transpose_38 = None + scores_14 += position_bias_3 + scores_15 = scores_14 + scores_14 = None + float_9 = scores_15.float() + softmax_7 = torch.nn.functional.softmax(float_9, dim=-1) + float_9 = None + attn_weights_14 = softmax_7.type_as(scores_15) + softmax_7 = scores_15 = None + attn_weights_15 = torch.nn.functional.dropout( + attn_weights_14, p=0.1, training=False + ) + attn_weights_14 = None + attn_output_28 = torch.matmul(attn_weights_15, value_states_15) + attn_weights_15 = None + transpose_39 = attn_output_28.transpose(1, 2) + attn_output_28 = None + attn_output_29 = transpose_39.contiguous() + transpose_39 = None + attn_output_30 = attn_output_29.view(1, -1, 512) + attn_output_29 = None + attn_output_31 = torch._C._nn.linear( + attn_output_30, + l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_30 = l_self_modules_block_modules_3_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_22 = torch.nn.functional.dropout(attn_output_31, 0.1, False, False) + attn_output_31 = None + layer_output_3 = hidden_states_29 + dropout_22 + hidden_states_29 = dropout_22 = None + to_14 = layer_output_3.to(torch.float32) + pow_12 = to_14.pow(2) + to_14 = None + variance_11 = pow_12.mean(-1, keepdim=True) + pow_12 = None + add_31 = variance_11 + 1e-06 + variance_11 = None + rsqrt_11 = torch.rsqrt(add_31) + add_31 = None + hidden_states_31 = layer_output_3 * rsqrt_11 + rsqrt_11 = None + forwarded_states_3 = ( + l_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_31 + ) + l_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_31 + ) = None + hidden_states_32 = torch._C._nn.linear( + forwarded_states_3, + l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states_3 = l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_33 = torch.nn.functional.relu(hidden_states_32, inplace=False) + hidden_states_32 = None + hidden_states_34 = torch.nn.functional.dropout( + hidden_states_33, 0.1, False, False + ) + hidden_states_33 = None + hidden_states_35 = torch._C._nn.linear( + hidden_states_34, + l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_34 = l_self_modules_block_modules_3_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_24 = torch.nn.functional.dropout(hidden_states_35, 0.1, False, False) + hidden_states_35 = None + hidden_states_36 = layer_output_3 + dropout_24 + layer_output_3 = dropout_24 = None + to_15 = hidden_states_36.to(torch.float32) + pow_13 = to_15.pow(2) + to_15 = None + variance_12 = pow_13.mean(-1, keepdim=True) + pow_13 = None + add_33 = variance_12 + 1e-06 + variance_12 = None + rsqrt_12 = torch.rsqrt(add_33) + add_33 = None + hidden_states_37 = hidden_states_36 * rsqrt_12 + rsqrt_12 = None + normed_hidden_states_8 = ( + l_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_37 + ) + l_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_37 + ) = None + query_states_16 = torch._C._nn.linear( + normed_hidden_states_8, + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view_32 = query_states_16.view(1, -1, 8, 64) + query_states_16 = None + query_states_17 = view_32.transpose(1, 2) + view_32 = None + key_states_16 = torch._C._nn.linear( + normed_hidden_states_8, + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_16 = torch._C._nn.linear( + normed_hidden_states_8, + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states_8 = l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_33 = key_states_16.view(1, -1, 8, 64) + key_states_16 = None + key_states_17 = view_33.transpose(1, 2) + view_33 = None + view_34 = value_states_16.view(1, -1, 8, 64) + value_states_16 = None + value_states_17 = view_34.transpose(1, 2) + view_34 = None + transpose_43 = key_states_17.transpose(3, 2) + scores_16 = torch.matmul(query_states_17, transpose_43) + query_states_17 = transpose_43 = None + scores_16 += position_bias_1 + scores_17 = scores_16 + scores_16 = None + float_10 = scores_17.float() + softmax_8 = torch.nn.functional.softmax(float_10, dim=-1) + float_10 = None + attn_weights_16 = softmax_8.type_as(scores_17) + softmax_8 = scores_17 = None + attn_weights_17 = torch.nn.functional.dropout( + attn_weights_16, p=0.1, training=False + ) + attn_weights_16 = None + attn_output_32 = torch.matmul(attn_weights_17, value_states_17) + attn_weights_17 = None + transpose_44 = attn_output_32.transpose(1, 2) + attn_output_32 = None + attn_output_33 = transpose_44.contiguous() + transpose_44 = None + attn_output_34 = attn_output_33.view(1, -1, 512) + attn_output_33 = None + attn_output_35 = torch._C._nn.linear( + attn_output_34, + l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_34 = l_self_modules_block_modules_4_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_26 = torch.nn.functional.dropout(attn_output_35, 0.1, False, False) + attn_output_35 = None + hidden_states_38 = hidden_states_36 + dropout_26 + hidden_states_36 = dropout_26 = None + getitem_12 = cache_position[-1] + add_35 = getitem_12 + 1 + getitem_12 = add_35 = None + to_16 = hidden_states_38.to(torch.float32) + pow_14 = to_16.pow(2) + to_16 = None + variance_13 = pow_14.mean(-1, keepdim=True) + pow_14 = None + add_36 = variance_13 + 1e-06 + variance_13 = None + rsqrt_13 = torch.rsqrt(add_36) + add_36 = None + hidden_states_39 = hidden_states_38 * rsqrt_13 + rsqrt_13 = None + normed_hidden_states_9 = ( + l_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_39 + ) + l_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_39 + ) = None + query_states_18 = torch._C._nn.linear( + normed_hidden_states_9, + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_9 = l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_36 = query_states_18.view(1, -1, 8, 64) + query_states_18 = None + query_states_19 = view_36.transpose(1, 2) + view_36 = None + key_states_18 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_18 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = ( + None + ) + view_37 = key_states_18.view(1, -1, 8, 64) + key_states_18 = None + key_states_19 = view_37.transpose(1, 2) + view_37 = None + view_38 = value_states_18.view(1, -1, 8, 64) + value_states_18 = None + value_states_19 = view_38.transpose(1, 2) + view_38 = None + transpose_48 = key_states_19.transpose(3, 2) + scores_18 = torch.matmul(query_states_19, transpose_48) + query_states_19 = transpose_48 = None + scores_18 += position_bias_3 + scores_19 = scores_18 + scores_18 = None + float_11 = scores_19.float() + softmax_9 = torch.nn.functional.softmax(float_11, dim=-1) + float_11 = None + attn_weights_18 = softmax_9.type_as(scores_19) + softmax_9 = scores_19 = None + attn_weights_19 = torch.nn.functional.dropout( + attn_weights_18, p=0.1, training=False + ) + attn_weights_18 = None + attn_output_36 = torch.matmul(attn_weights_19, value_states_19) + attn_weights_19 = None + transpose_49 = attn_output_36.transpose(1, 2) + attn_output_36 = None + attn_output_37 = transpose_49.contiguous() + transpose_49 = None + attn_output_38 = attn_output_37.view(1, -1, 512) + attn_output_37 = None + attn_output_39 = torch._C._nn.linear( + attn_output_38, + l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_38 = l_self_modules_block_modules_4_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_28 = torch.nn.functional.dropout(attn_output_39, 0.1, False, False) + attn_output_39 = None + layer_output_4 = hidden_states_38 + dropout_28 + hidden_states_38 = dropout_28 = None + to_17 = layer_output_4.to(torch.float32) + pow_15 = to_17.pow(2) + to_17 = None + variance_14 = pow_15.mean(-1, keepdim=True) + pow_15 = None + add_38 = variance_14 + 1e-06 + variance_14 = None + rsqrt_14 = torch.rsqrt(add_38) + add_38 = None + hidden_states_40 = layer_output_4 * rsqrt_14 + rsqrt_14 = None + forwarded_states_4 = ( + l_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_40 + ) + l_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_40 + ) = None + hidden_states_41 = torch._C._nn.linear( + forwarded_states_4, + l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states_4 = l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_42 = torch.nn.functional.relu(hidden_states_41, inplace=False) + hidden_states_41 = None + hidden_states_43 = torch.nn.functional.dropout( + hidden_states_42, 0.1, False, False + ) + hidden_states_42 = None + hidden_states_44 = torch._C._nn.linear( + hidden_states_43, + l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_43 = l_self_modules_block_modules_4_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_30 = torch.nn.functional.dropout(hidden_states_44, 0.1, False, False) + hidden_states_44 = None + hidden_states_45 = layer_output_4 + dropout_30 + layer_output_4 = dropout_30 = None + to_18 = hidden_states_45.to(torch.float32) + pow_16 = to_18.pow(2) + to_18 = None + variance_15 = pow_16.mean(-1, keepdim=True) + pow_16 = None + add_40 = variance_15 + 1e-06 + variance_15 = None + rsqrt_15 = torch.rsqrt(add_40) + add_40 = None + hidden_states_46 = hidden_states_45 * rsqrt_15 + rsqrt_15 = None + normed_hidden_states_10 = ( + l_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_ + * hidden_states_46 + ) + l_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_ = ( + hidden_states_46 + ) = None + query_states_20 = torch._C._nn.linear( + normed_hidden_states_10, + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_, + None, + ) + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_q_parameters_weight_ = ( + None + ) + view_40 = query_states_20.view(1, -1, 8, 64) + query_states_20 = None + query_states_21 = view_40.transpose(1, 2) + view_40 = None + key_states_20 = torch._C._nn.linear( + normed_hidden_states_10, + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_20 = torch._C._nn.linear( + normed_hidden_states_10, + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_, + None, + ) + normed_hidden_states_10 = l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_v_parameters_weight_ = (None) + view_41 = key_states_20.view(1, -1, 8, 64) + key_states_20 = None + key_states_21 = view_41.transpose(1, 2) + view_41 = None + view_42 = value_states_20.view(1, -1, 8, 64) + value_states_20 = None + value_states_21 = view_42.transpose(1, 2) + view_42 = None + transpose_53 = key_states_21.transpose(3, 2) + scores_20 = torch.matmul(query_states_21, transpose_53) + query_states_21 = transpose_53 = None + scores_20 += position_bias_1 + scores_21 = scores_20 + scores_20 = position_bias_1 = None + float_12 = scores_21.float() + softmax_10 = torch.nn.functional.softmax(float_12, dim=-1) + float_12 = None + attn_weights_20 = softmax_10.type_as(scores_21) + softmax_10 = scores_21 = None + attn_weights_21 = torch.nn.functional.dropout( + attn_weights_20, p=0.1, training=False + ) + attn_weights_20 = None + attn_output_40 = torch.matmul(attn_weights_21, value_states_21) + attn_weights_21 = None + transpose_54 = attn_output_40.transpose(1, 2) + attn_output_40 = None + attn_output_41 = transpose_54.contiguous() + transpose_54 = None + attn_output_42 = attn_output_41.view(1, -1, 512) + attn_output_41 = None + attn_output_43 = torch._C._nn.linear( + attn_output_42, + l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_, + None, + ) + attn_output_42 = l_self_modules_block_modules_5_modules_layer_modules_0_modules_self_attention_modules_o_parameters_weight_ = (None) + dropout_32 = torch.nn.functional.dropout(attn_output_43, 0.1, False, False) + attn_output_43 = None + hidden_states_47 = hidden_states_45 + dropout_32 + hidden_states_45 = dropout_32 = None + getitem_13 = cache_position[-1] + cache_position = None + add_42 = getitem_13 + 1 + getitem_13 = add_42 = None + to_19 = hidden_states_47.to(torch.float32) + pow_17 = to_19.pow(2) + to_19 = None + variance_16 = pow_17.mean(-1, keepdim=True) + pow_17 = None + add_43 = variance_16 + 1e-06 + variance_16 = None + rsqrt_16 = torch.rsqrt(add_43) + add_43 = None + hidden_states_48 = hidden_states_47 * rsqrt_16 + rsqrt_16 = None + normed_hidden_states_11 = ( + l_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_ + * hidden_states_48 + ) + l_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_ = ( + hidden_states_48 + ) = None + query_states_22 = torch._C._nn.linear( + normed_hidden_states_11, + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_, + None, + ) + normed_hidden_states_11 = l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_q_parameters_weight_ = (None) + view_44 = query_states_22.view(1, -1, 8, 64) + query_states_22 = None + query_states_23 = view_44.transpose(1, 2) + view_44 = None + key_states_22 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_, + None, + ) + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_k_parameters_weight_ = ( + None + ) + value_states_22 = torch._C._nn.linear( + l_encoder_hidden_states_, + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_, + None, + ) + l_encoder_hidden_states_ = l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_v_parameters_weight_ = (None) + view_45 = key_states_22.view(1, -1, 8, 64) + key_states_22 = None + key_states_23 = view_45.transpose(1, 2) + view_45 = None + view_46 = value_states_22.view(1, -1, 8, 64) + value_states_22 = None + value_states_23 = view_46.transpose(1, 2) + view_46 = None + transpose_58 = key_states_23.transpose(3, 2) + scores_22 = torch.matmul(query_states_23, transpose_58) + query_states_23 = transpose_58 = None + scores_22 += position_bias_3 + scores_23 = scores_22 + scores_22 = position_bias_3 = None + float_13 = scores_23.float() + softmax_11 = torch.nn.functional.softmax(float_13, dim=-1) + float_13 = None + attn_weights_22 = softmax_11.type_as(scores_23) + softmax_11 = scores_23 = None + attn_weights_23 = torch.nn.functional.dropout( + attn_weights_22, p=0.1, training=False + ) + attn_weights_22 = None + attn_output_44 = torch.matmul(attn_weights_23, value_states_23) + attn_weights_23 = None + transpose_59 = attn_output_44.transpose(1, 2) + attn_output_44 = None + attn_output_45 = transpose_59.contiguous() + transpose_59 = None + attn_output_46 = attn_output_45.view(1, -1, 512) + attn_output_45 = None + attn_output_47 = torch._C._nn.linear( + attn_output_46, + l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_, + None, + ) + attn_output_46 = l_self_modules_block_modules_5_modules_layer_modules_1_modules_enc_dec_attention_modules_o_parameters_weight_ = (None) + dropout_34 = torch.nn.functional.dropout(attn_output_47, 0.1, False, False) + attn_output_47 = None + layer_output_5 = hidden_states_47 + dropout_34 + hidden_states_47 = dropout_34 = None + to_20 = layer_output_5.to(torch.float32) + pow_18 = to_20.pow(2) + to_20 = None + variance_17 = pow_18.mean(-1, keepdim=True) + pow_18 = None + add_45 = variance_17 + 1e-06 + variance_17 = None + rsqrt_17 = torch.rsqrt(add_45) + add_45 = None + hidden_states_49 = layer_output_5 * rsqrt_17 + rsqrt_17 = None + forwarded_states_5 = ( + l_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_ + * hidden_states_49 + ) + l_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_ = ( + hidden_states_49 + ) = None + hidden_states_50 = torch._C._nn.linear( + forwarded_states_5, + l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_, + None, + ) + forwarded_states_5 = l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wi_parameters_weight_ = (None) + hidden_states_51 = torch.nn.functional.relu(hidden_states_50, inplace=False) + hidden_states_50 = None + hidden_states_52 = torch.nn.functional.dropout( + hidden_states_51, 0.1, False, False + ) + hidden_states_51 = None + hidden_states_53 = torch._C._nn.linear( + hidden_states_52, + l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_, + None, + ) + hidden_states_52 = l_self_modules_block_modules_5_modules_layer_modules_2_modules_dense_relu_dense_modules_wo_parameters_weight_ = (None) + dropout_36 = torch.nn.functional.dropout(hidden_states_53, 0.1, False, False) + hidden_states_53 = None + hidden_states_54 = layer_output_5 + dropout_36 + layer_output_5 = dropout_36 = None + to_21 = hidden_states_54.to(torch.float32) + pow_19 = to_21.pow(2) + to_21 = None + variance_18 = pow_19.mean(-1, keepdim=True) + pow_19 = None + add_47 = variance_18 + 1e-06 + variance_18 = None + rsqrt_18 = torch.rsqrt(add_47) + add_47 = None + hidden_states_55 = hidden_states_54 * rsqrt_18 + hidden_states_54 = rsqrt_18 = None + hidden_states_56 = ( + l_self_modules_final_layer_norm_parameters_weight_ * hidden_states_55 + ) + l_self_modules_final_layer_norm_parameters_weight_ = hidden_states_55 = None + hidden_states_57 = torch.nn.functional.dropout( + hidden_states_56, 0.1, False, False + ) + hidden_states_56 = None + return ( + value_states_1, + key_states_1, + value_states_3, + key_states_3, + value_states_5, + key_states_5, + value_states_7, + key_states_7, + value_states_9, + key_states_9, + value_states_11, + key_states_11, + value_states_13, + key_states_13, + value_states_15, + key_states_15, + value_states_17, + key_states_17, + value_states_19, + key_states_19, + value_states_21, + key_states_21, + value_states_23, + key_states_23, + hidden_states_57, + ) diff --git a/samples/transformers-auto-model/t5-small/weight_meta.py b/samples/transformers-auto-model/t5-small/weight_meta.py new file mode 100644 index 0000000000..118b8ab812 --- /dev/null +++ b/samples/transformers-auto-model/t5-small/weight_meta.py @@ -0,0 +1,828 @@ +class Program_weight_tensor_meta_L_inputs_embeds_: + name = "L_inputs_embeds_" + shape = [1, 12, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -1.734 + std = 37.465 + data = None + + +class Program_weight_tensor_meta_L_encoder_hidden_states_: + name = "L_encoder_hidden_states_" + shape = [1, 12, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.146 + data = None + + +class Program_weight_tensor_meta_L_encoder_attention_mask_: + name = "L_encoder_attention_mask_" + shape = [1, 12] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.089 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.070 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.560 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.490 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_relative_attention_bias_parameters_weight_" + shape = [32, 8] + dtype = "torch.float32" + device = "cuda:0" + mean = -1.320 + std = 7.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.857 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.089 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.060 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.393 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.469 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.717 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.675 + std = 0.327 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.015 + std = 0.620 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_0_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.335 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.131 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.429 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.724 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.912 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.097 + std = 0.025 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.074 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.351 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.636 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.651 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.160 + std = 0.229 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.025 + std = 0.714 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_1_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.359 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.143 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.407 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.908 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 1.072 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.088 + std = 0.021 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.392 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.739 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.734 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.451 + std = 0.242 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.020 + std = 0.737 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_2_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.388 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.155 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.046 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.419 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.958 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 1.434 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.120 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.410 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.921 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.895 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.923 + std = 0.349 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.014 + std = 0.680 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_3_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.399 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.173 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.411 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.892 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 2.320 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.072 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.056 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.479 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 1.220 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 1.269 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 2.439 + std = 0.694 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.694 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_4_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.420 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_0_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.221 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.395 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.003 + std = 0.954 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_0_modules_SelfAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 1.539 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_1_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.142 + std = 0.098 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_q_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_k_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.387 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_v_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 1.798 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_1_modules_EncDecAttention_modules_o_parameters_weight_" + shape = [512, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 1.473 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_2_modules_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 3.056 + std = 1.259 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wi_parameters_weight_" + shape = [2048, 512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.821 + data = None + + +class Program_weight_tensor_meta_L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_: + name = "L_self_modules_block_modules_5_modules_layer_modules_2_modules_DenseReluDense_modules_wo_parameters_weight_" + shape = [512, 2048] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.469 + data = None + + +class Program_weight_tensor_meta_L_self_modules_final_layer_norm_parameters_weight_: + name = "L_self_modules_final_layer_norm_parameters_weight_" + shape = [512] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.239 + std = 0.529 + data = None diff --git a/samples/transformers-auto-model/xlnet-base-cased/graph_hash.txt b/samples/transformers-auto-model/xlnet-base-cased/graph_hash.txt new file mode 100644 index 0000000000..d5c96f36f7 --- /dev/null +++ b/samples/transformers-auto-model/xlnet-base-cased/graph_hash.txt @@ -0,0 +1 @@ +82c6427c42ff7acfbcc1ccec42d26cc1e0b99754c48fb9767b773a6665b60bd6 \ No newline at end of file diff --git a/samples/transformers-auto-model/xlnet-base-cased/graph_net.json b/samples/transformers-auto-model/xlnet-base-cased/graph_net.json new file mode 100644 index 0000000000..1373fe3b58 --- /dev/null +++ b/samples/transformers-auto-model/xlnet-base-cased/graph_net.json @@ -0,0 +1,5 @@ +{ + "framework": "torch", + "num_devices_required": 1, + "num_nodes_required": 1 +} \ No newline at end of file diff --git a/samples/transformers-auto-model/xlnet-base-cased/input_meta.py b/samples/transformers-auto-model/xlnet-base-cased/input_meta.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/xlnet-base-cased/input_tensor_constraints.py b/samples/transformers-auto-model/xlnet-base-cased/input_tensor_constraints.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/transformers-auto-model/xlnet-base-cased/model.py b/samples/transformers-auto-model/xlnet-base-cased/model.py new file mode 100644 index 0000000000..687d2b6c96 --- /dev/null +++ b/samples/transformers-auto-model/xlnet-base-cased/model.py @@ -0,0 +1,2632 @@ +import torch + +from torch import device + + +class GraphModule(torch.nn.Module): + def forward( + self, + L_input_ids_: torch.Tensor, + L_token_type_ids_: torch.Tensor, + L_attention_mask_: torch.Tensor, + L_self_modules_word_embedding_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_q_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_k_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_v_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_parameters_o_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_: torch.nn.parameter.Parameter, + L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_: torch.nn.parameter.Parameter, + ): + l_input_ids_ = L_input_ids_ + l_token_type_ids_ = L_token_type_ids_ + l_attention_mask_ = L_attention_mask_ + l_self_modules_word_embedding_parameters_weight_ = ( + L_self_modules_word_embedding_parameters_weight_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_0_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_1_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_1_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_2_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_2_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_3_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_3_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_4_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_4_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_5_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_5_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_6_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_6_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_7_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_7_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_8_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_8_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_9_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_9_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_ = ( + L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_ + ) + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_ = ( + L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_ + ) + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_10_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_10_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_ = L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_ + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_ = L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_ + l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_11_modules_rel_attn_parameters_q_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_q_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_k_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_k_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_v_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_v_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_o_ = ( + L_self_modules_layer_modules_11_modules_rel_attn_parameters_o_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_ + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_ = L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_ + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_ = ( + L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_ + ) + l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_ = L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_ + l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_ = ( + L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_ + ) + l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_ = L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_ + l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_ = L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_ + transpose = l_input_ids_.transpose(0, 1) + l_input_ids_ = None + input_ids = transpose.contiguous() + transpose = None + transpose_1 = l_token_type_ids_.transpose(0, 1) + l_token_type_ids_ = None + token_type_ids = transpose_1.contiguous() + transpose_1 = None + transpose_2 = l_attention_mask_.transpose(0, 1) + l_attention_mask_ = None + attention_mask = transpose_2.contiguous() + transpose_2 = None + input_mask = 1.0 - attention_mask + attention_mask = None + data_mask = input_mask[None] + input_mask = None + attn_mask = data_mask[ + ( + slice(None, None, None), + slice(None, None, None), + slice(None, None, None), + None, + ) + ] + data_mask = None + gt = attn_mask > 0 + attn_mask = None + attn_mask_1 = gt.to(torch.float32) + gt = None + eye = torch.eye(13) + to_1 = eye.to(attn_mask_1) + eye = None + non_tgt_mask = -to_1 + to_1 = None + getitem_2 = non_tgt_mask[ + (slice(None, None, None), slice(None, None, None), None, None) + ] + non_tgt_mask = None + add = attn_mask_1 + getitem_2 + getitem_2 = None + gt_1 = add > 0 + add = None + non_tgt_mask_1 = gt_1.to(attn_mask_1) + gt_1 = attn_mask_1 = None + word_emb_k = torch.nn.functional.embedding( + input_ids, + l_self_modules_word_embedding_parameters_weight_, + None, + None, + 2.0, + False, + False, + ) + input_ids = l_self_modules_word_embedding_parameters_weight_ = None + output_h = torch.nn.functional.dropout(word_emb_k, 0.1, False, False) + word_emb_k = None + getitem_3 = token_type_ids[(slice(None, None, None), None)] + getitem_4 = token_type_ids[(None, slice(None, None, None))] + token_type_ids = None + ne = getitem_3 != getitem_4 + getitem_3 = getitem_4 = None + seg_mat = ne.long() + ne = None + one_hot = torch._C._nn.one_hot(seg_mat, num_classes=2) + seg_mat = None + seg_mat_1 = one_hot.to(torch.float32) + one_hot = None + arange = torch.arange(0, 768, 2.0, dtype=torch.int64) + freq_seq = arange.float() + arange = None + truediv = freq_seq / 768 + freq_seq = None + pow_1 = torch.pow(10000, truediv) + truediv = None + inv_freq = 1 / pow_1 + pow_1 = None + arange_1 = torch.arange(13, -13, -1.0, dtype=torch.int64) + fwd_pos_seq = arange_1.float() + arange_1 = None + sinusoid_inp = torch.functional.einsum("i,d->id", fwd_pos_seq, inv_freq) + fwd_pos_seq = inv_freq = None + sin = torch.sin(sinusoid_inp) + cos = torch.cos(sinusoid_inp) + sinusoid_inp = None + pos_emb = torch.cat([sin, cos], dim=-1) + sin = cos = None + pos_emb_1 = pos_emb[(slice(None, None, None), None, slice(None, None, None))] + pos_emb = None + pos_emb_2 = pos_emb_1.expand(-1, 1, -1) + pos_emb_1 = None + pos_emb_3 = pos_emb_2.to(device(type="cuda", index=0)) + pos_emb_2 = None + pos_emb_4 = torch.nn.functional.dropout(pos_emb_3, 0.1, False, False) + pos_emb_3 = None + new_mem = output_h[slice(0, None, None)] + detach = new_mem.detach() + new_mem = None + q_head_h = torch.functional.einsum( + "ibh,hnd->ibnd", + output_h, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_q_ = None + k_head_h = torch.functional.einsum( + "ibh,hnd->ibnd", + output_h, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_k_ = None + v_head_h = torch.functional.einsum( + "ibh,hnd->ibnd", + output_h, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_v_ = None + type_1 = pos_emb_4.type(torch.float32) + k_head_r = torch.functional.einsum( + "ibh,hnd->ibnd", + type_1, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_, + ) + type_1 = l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_ = None + add_1 = ( + q_head_h + + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_ = None + ac = torch.functional.einsum("ibnd,jbnd->bnij", add_1, k_head_h) + add_1 = k_head_h = None + add_2 = ( + q_head_h + + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_ = None + bd = torch.functional.einsum("ibnd,jbnd->bnij", add_2, k_head_r) + add_2 = k_head_r = None + x = bd.reshape(1, 12, 26, 13) + bd = None + x_1 = x[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x = None + x_2 = x_1.reshape(1, 12, 13, 25) + x_1 = None + arange_2 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_3 = torch.index_select(x_2, 3, arange_2) + x_2 = arange_2 = None + add_3 = ( + q_head_h + + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h = ( + l_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef = torch.functional.einsum( + "ibnd,snd->ibns", + add_3, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_, + ) + add_3 = ( + l_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_1 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef) + ef = None + add_4 = ac + x_3 + ac = x_3 = None + add_5 = add_4 + ef_1 + add_4 = ef_1 = None + attn_score = add_5 * 0.125 + add_5 = None + einsum_9 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_1 = 1e30 * einsum_9 + einsum_9 = None + attn_score_1 = attn_score - mul_1 + attn_score = mul_1 = None + attn_prob = torch.nn.functional.softmax(attn_score_1, dim=3) + attn_score_1 = None + attn_prob_1 = torch.nn.functional.dropout(attn_prob, 0.1, False, False) + attn_prob = None + attn_vec = torch.functional.einsum("bnij,jbnd->ibnd", attn_prob_1, v_head_h) + attn_prob_1 = v_head_h = None + attn_out = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec, + l_self_modules_layer_modules_0_modules_rel_attn_parameters_o_, + ) + attn_vec = l_self_modules_layer_modules_0_modules_rel_attn_parameters_o_ = None + attn_out_1 = torch.nn.functional.dropout(attn_out, 0.1, False, False) + attn_out = None + attn_out_2 = attn_out_1 + output_h + attn_out_1 = output_h = None + output = torch.nn.functional.layer_norm( + attn_out_2, + (768,), + l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_2 = l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_1 = torch._C._nn.linear( + output, + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_2 = torch._C._nn.gelu(output_1) + output_1 = None + output_3 = torch.nn.functional.dropout(output_2, 0.1, False, False) + output_2 = None + output_4 = torch._C._nn.linear( + output_3, + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_, + ) + output_3 = ( + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_5 = torch.nn.functional.dropout(output_4, 0.1, False, False) + output_4 = None + add_7 = output_5 + output + output_5 = output = None + output_6 = torch.nn.functional.layer_norm( + add_7, + (768,), + l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_7 = l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_1 = output_6[slice(0, None, None)] + detach_1 = new_mem_1.detach() + new_mem_1 = None + q_head_h_1 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_6, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_q_ = None + k_head_h_1 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_6, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_k_ = None + v_head_h_1 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_6, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_v_ = None + type_2 = pos_emb_4.type(torch.float32) + k_head_r_1 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_2, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_, + ) + type_2 = l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_ = None + add_8 = ( + q_head_h_1 + + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_ = None + ac_1 = torch.functional.einsum("ibnd,jbnd->bnij", add_8, k_head_h_1) + add_8 = k_head_h_1 = None + add_9 = ( + q_head_h_1 + + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_ = None + bd_1 = torch.functional.einsum("ibnd,jbnd->bnij", add_9, k_head_r_1) + add_9 = k_head_r_1 = None + x_4 = bd_1.reshape(1, 12, 26, 13) + bd_1 = None + x_5 = x_4[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_4 = None + x_6 = x_5.reshape(1, 12, 13, 25) + x_5 = None + arange_3 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_7 = torch.index_select(x_6, 3, arange_3) + x_6 = arange_3 = None + add_10 = ( + q_head_h_1 + + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_1 = ( + l_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_2 = torch.functional.einsum( + "ibnd,snd->ibns", + add_10, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_, + ) + add_10 = ( + l_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_3 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_2) + ef_2 = None + add_11 = ac_1 + x_7 + ac_1 = x_7 = None + add_12 = add_11 + ef_3 + add_11 = ef_3 = None + attn_score_2 = add_12 * 0.125 + add_12 = None + einsum_20 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_3 = 1e30 * einsum_20 + einsum_20 = None + attn_score_3 = attn_score_2 - mul_3 + attn_score_2 = mul_3 = None + attn_prob_2 = torch.nn.functional.softmax(attn_score_3, dim=3) + attn_score_3 = None + attn_prob_3 = torch.nn.functional.dropout(attn_prob_2, 0.1, False, False) + attn_prob_2 = None + attn_vec_1 = torch.functional.einsum("bnij,jbnd->ibnd", attn_prob_3, v_head_h_1) + attn_prob_3 = v_head_h_1 = None + attn_out_3 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_1, + l_self_modules_layer_modules_1_modules_rel_attn_parameters_o_, + ) + attn_vec_1 = ( + l_self_modules_layer_modules_1_modules_rel_attn_parameters_o_ + ) = None + attn_out_4 = torch.nn.functional.dropout(attn_out_3, 0.1, False, False) + attn_out_3 = None + attn_out_5 = attn_out_4 + output_6 + attn_out_4 = output_6 = None + output_7 = torch.nn.functional.layer_norm( + attn_out_5, + (768,), + l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_5 = l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_8 = torch._C._nn.linear( + output_7, + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_9 = torch._C._nn.gelu(output_8) + output_8 = None + output_10 = torch.nn.functional.dropout(output_9, 0.1, False, False) + output_9 = None + output_11 = torch._C._nn.linear( + output_10, + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_, + ) + output_10 = ( + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_12 = torch.nn.functional.dropout(output_11, 0.1, False, False) + output_11 = None + add_14 = output_12 + output_7 + output_12 = output_7 = None + output_13 = torch.nn.functional.layer_norm( + add_14, + (768,), + l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_14 = l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_2 = output_13[slice(0, None, None)] + detach_2 = new_mem_2.detach() + new_mem_2 = None + q_head_h_2 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_13, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_q_ = None + k_head_h_2 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_13, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_k_ = None + v_head_h_2 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_13, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_v_ = None + type_3 = pos_emb_4.type(torch.float32) + k_head_r_2 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_3, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_, + ) + type_3 = l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_ = None + add_15 = ( + q_head_h_2 + + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_ = None + ac_2 = torch.functional.einsum("ibnd,jbnd->bnij", add_15, k_head_h_2) + add_15 = k_head_h_2 = None + add_16 = ( + q_head_h_2 + + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_ = None + bd_2 = torch.functional.einsum("ibnd,jbnd->bnij", add_16, k_head_r_2) + add_16 = k_head_r_2 = None + x_8 = bd_2.reshape(1, 12, 26, 13) + bd_2 = None + x_9 = x_8[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_8 = None + x_10 = x_9.reshape(1, 12, 13, 25) + x_9 = None + arange_4 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_11 = torch.index_select(x_10, 3, arange_4) + x_10 = arange_4 = None + add_17 = ( + q_head_h_2 + + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_2 = ( + l_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_4 = torch.functional.einsum( + "ibnd,snd->ibns", + add_17, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_, + ) + add_17 = ( + l_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_5 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_4) + ef_4 = None + add_18 = ac_2 + x_11 + ac_2 = x_11 = None + add_19 = add_18 + ef_5 + add_18 = ef_5 = None + attn_score_4 = add_19 * 0.125 + add_19 = None + einsum_31 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_5 = 1e30 * einsum_31 + einsum_31 = None + attn_score_5 = attn_score_4 - mul_5 + attn_score_4 = mul_5 = None + attn_prob_4 = torch.nn.functional.softmax(attn_score_5, dim=3) + attn_score_5 = None + attn_prob_5 = torch.nn.functional.dropout(attn_prob_4, 0.1, False, False) + attn_prob_4 = None + attn_vec_2 = torch.functional.einsum("bnij,jbnd->ibnd", attn_prob_5, v_head_h_2) + attn_prob_5 = v_head_h_2 = None + attn_out_6 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_2, + l_self_modules_layer_modules_2_modules_rel_attn_parameters_o_, + ) + attn_vec_2 = ( + l_self_modules_layer_modules_2_modules_rel_attn_parameters_o_ + ) = None + attn_out_7 = torch.nn.functional.dropout(attn_out_6, 0.1, False, False) + attn_out_6 = None + attn_out_8 = attn_out_7 + output_13 + attn_out_7 = output_13 = None + output_14 = torch.nn.functional.layer_norm( + attn_out_8, + (768,), + l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_8 = l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_15 = torch._C._nn.linear( + output_14, + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_16 = torch._C._nn.gelu(output_15) + output_15 = None + output_17 = torch.nn.functional.dropout(output_16, 0.1, False, False) + output_16 = None + output_18 = torch._C._nn.linear( + output_17, + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_, + ) + output_17 = ( + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_19 = torch.nn.functional.dropout(output_18, 0.1, False, False) + output_18 = None + add_21 = output_19 + output_14 + output_19 = output_14 = None + output_20 = torch.nn.functional.layer_norm( + add_21, + (768,), + l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_21 = l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_3 = output_20[slice(0, None, None)] + detach_3 = new_mem_3.detach() + new_mem_3 = None + q_head_h_3 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_20, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_q_ = None + k_head_h_3 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_20, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_k_ = None + v_head_h_3 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_20, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_v_ = None + type_4 = pos_emb_4.type(torch.float32) + k_head_r_3 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_4, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_, + ) + type_4 = l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_ = None + add_22 = ( + q_head_h_3 + + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_ = None + ac_3 = torch.functional.einsum("ibnd,jbnd->bnij", add_22, k_head_h_3) + add_22 = k_head_h_3 = None + add_23 = ( + q_head_h_3 + + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_ = None + bd_3 = torch.functional.einsum("ibnd,jbnd->bnij", add_23, k_head_r_3) + add_23 = k_head_r_3 = None + x_12 = bd_3.reshape(1, 12, 26, 13) + bd_3 = None + x_13 = x_12[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_12 = None + x_14 = x_13.reshape(1, 12, 13, 25) + x_13 = None + arange_5 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_15 = torch.index_select(x_14, 3, arange_5) + x_14 = arange_5 = None + add_24 = ( + q_head_h_3 + + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_3 = ( + l_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_6 = torch.functional.einsum( + "ibnd,snd->ibns", + add_24, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_, + ) + add_24 = ( + l_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_7 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_6) + ef_6 = None + add_25 = ac_3 + x_15 + ac_3 = x_15 = None + add_26 = add_25 + ef_7 + add_25 = ef_7 = None + attn_score_6 = add_26 * 0.125 + add_26 = None + einsum_42 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_7 = 1e30 * einsum_42 + einsum_42 = None + attn_score_7 = attn_score_6 - mul_7 + attn_score_6 = mul_7 = None + attn_prob_6 = torch.nn.functional.softmax(attn_score_7, dim=3) + attn_score_7 = None + attn_prob_7 = torch.nn.functional.dropout(attn_prob_6, 0.1, False, False) + attn_prob_6 = None + attn_vec_3 = torch.functional.einsum("bnij,jbnd->ibnd", attn_prob_7, v_head_h_3) + attn_prob_7 = v_head_h_3 = None + attn_out_9 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_3, + l_self_modules_layer_modules_3_modules_rel_attn_parameters_o_, + ) + attn_vec_3 = ( + l_self_modules_layer_modules_3_modules_rel_attn_parameters_o_ + ) = None + attn_out_10 = torch.nn.functional.dropout(attn_out_9, 0.1, False, False) + attn_out_9 = None + attn_out_11 = attn_out_10 + output_20 + attn_out_10 = output_20 = None + output_21 = torch.nn.functional.layer_norm( + attn_out_11, + (768,), + l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_11 = l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_22 = torch._C._nn.linear( + output_21, + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_23 = torch._C._nn.gelu(output_22) + output_22 = None + output_24 = torch.nn.functional.dropout(output_23, 0.1, False, False) + output_23 = None + output_25 = torch._C._nn.linear( + output_24, + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_, + ) + output_24 = ( + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_26 = torch.nn.functional.dropout(output_25, 0.1, False, False) + output_25 = None + add_28 = output_26 + output_21 + output_26 = output_21 = None + output_27 = torch.nn.functional.layer_norm( + add_28, + (768,), + l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_28 = l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_4 = output_27[slice(0, None, None)] + detach_4 = new_mem_4.detach() + new_mem_4 = None + q_head_h_4 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_27, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_q_ = None + k_head_h_4 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_27, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_k_ = None + v_head_h_4 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_27, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_v_ = None + type_5 = pos_emb_4.type(torch.float32) + k_head_r_4 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_5, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_, + ) + type_5 = l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_ = None + add_29 = ( + q_head_h_4 + + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_ = None + ac_4 = torch.functional.einsum("ibnd,jbnd->bnij", add_29, k_head_h_4) + add_29 = k_head_h_4 = None + add_30 = ( + q_head_h_4 + + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_ = None + bd_4 = torch.functional.einsum("ibnd,jbnd->bnij", add_30, k_head_r_4) + add_30 = k_head_r_4 = None + x_16 = bd_4.reshape(1, 12, 26, 13) + bd_4 = None + x_17 = x_16[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_16 = None + x_18 = x_17.reshape(1, 12, 13, 25) + x_17 = None + arange_6 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_19 = torch.index_select(x_18, 3, arange_6) + x_18 = arange_6 = None + add_31 = ( + q_head_h_4 + + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_4 = ( + l_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_8 = torch.functional.einsum( + "ibnd,snd->ibns", + add_31, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_, + ) + add_31 = ( + l_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_9 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_8) + ef_8 = None + add_32 = ac_4 + x_19 + ac_4 = x_19 = None + add_33 = add_32 + ef_9 + add_32 = ef_9 = None + attn_score_8 = add_33 * 0.125 + add_33 = None + einsum_53 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_9 = 1e30 * einsum_53 + einsum_53 = None + attn_score_9 = attn_score_8 - mul_9 + attn_score_8 = mul_9 = None + attn_prob_8 = torch.nn.functional.softmax(attn_score_9, dim=3) + attn_score_9 = None + attn_prob_9 = torch.nn.functional.dropout(attn_prob_8, 0.1, False, False) + attn_prob_8 = None + attn_vec_4 = torch.functional.einsum("bnij,jbnd->ibnd", attn_prob_9, v_head_h_4) + attn_prob_9 = v_head_h_4 = None + attn_out_12 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_4, + l_self_modules_layer_modules_4_modules_rel_attn_parameters_o_, + ) + attn_vec_4 = ( + l_self_modules_layer_modules_4_modules_rel_attn_parameters_o_ + ) = None + attn_out_13 = torch.nn.functional.dropout(attn_out_12, 0.1, False, False) + attn_out_12 = None + attn_out_14 = attn_out_13 + output_27 + attn_out_13 = output_27 = None + output_28 = torch.nn.functional.layer_norm( + attn_out_14, + (768,), + l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_14 = l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_29 = torch._C._nn.linear( + output_28, + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_30 = torch._C._nn.gelu(output_29) + output_29 = None + output_31 = torch.nn.functional.dropout(output_30, 0.1, False, False) + output_30 = None + output_32 = torch._C._nn.linear( + output_31, + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_, + ) + output_31 = ( + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_33 = torch.nn.functional.dropout(output_32, 0.1, False, False) + output_32 = None + add_35 = output_33 + output_28 + output_33 = output_28 = None + output_34 = torch.nn.functional.layer_norm( + add_35, + (768,), + l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_35 = l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_5 = output_34[slice(0, None, None)] + detach_5 = new_mem_5.detach() + new_mem_5 = None + q_head_h_5 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_34, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_q_ = None + k_head_h_5 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_34, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_k_ = None + v_head_h_5 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_34, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_v_ = None + type_6 = pos_emb_4.type(torch.float32) + k_head_r_5 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_6, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_, + ) + type_6 = l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_ = None + add_36 = ( + q_head_h_5 + + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_ = None + ac_5 = torch.functional.einsum("ibnd,jbnd->bnij", add_36, k_head_h_5) + add_36 = k_head_h_5 = None + add_37 = ( + q_head_h_5 + + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_ = None + bd_5 = torch.functional.einsum("ibnd,jbnd->bnij", add_37, k_head_r_5) + add_37 = k_head_r_5 = None + x_20 = bd_5.reshape(1, 12, 26, 13) + bd_5 = None + x_21 = x_20[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_20 = None + x_22 = x_21.reshape(1, 12, 13, 25) + x_21 = None + arange_7 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_23 = torch.index_select(x_22, 3, arange_7) + x_22 = arange_7 = None + add_38 = ( + q_head_h_5 + + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_5 = ( + l_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_10 = torch.functional.einsum( + "ibnd,snd->ibns", + add_38, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_, + ) + add_38 = ( + l_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_11 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_10) + ef_10 = None + add_39 = ac_5 + x_23 + ac_5 = x_23 = None + add_40 = add_39 + ef_11 + add_39 = ef_11 = None + attn_score_10 = add_40 * 0.125 + add_40 = None + einsum_64 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_11 = 1e30 * einsum_64 + einsum_64 = None + attn_score_11 = attn_score_10 - mul_11 + attn_score_10 = mul_11 = None + attn_prob_10 = torch.nn.functional.softmax(attn_score_11, dim=3) + attn_score_11 = None + attn_prob_11 = torch.nn.functional.dropout(attn_prob_10, 0.1, False, False) + attn_prob_10 = None + attn_vec_5 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_11, v_head_h_5 + ) + attn_prob_11 = v_head_h_5 = None + attn_out_15 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_5, + l_self_modules_layer_modules_5_modules_rel_attn_parameters_o_, + ) + attn_vec_5 = ( + l_self_modules_layer_modules_5_modules_rel_attn_parameters_o_ + ) = None + attn_out_16 = torch.nn.functional.dropout(attn_out_15, 0.1, False, False) + attn_out_15 = None + attn_out_17 = attn_out_16 + output_34 + attn_out_16 = output_34 = None + output_35 = torch.nn.functional.layer_norm( + attn_out_17, + (768,), + l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_17 = l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_36 = torch._C._nn.linear( + output_35, + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_37 = torch._C._nn.gelu(output_36) + output_36 = None + output_38 = torch.nn.functional.dropout(output_37, 0.1, False, False) + output_37 = None + output_39 = torch._C._nn.linear( + output_38, + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_, + ) + output_38 = ( + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_40 = torch.nn.functional.dropout(output_39, 0.1, False, False) + output_39 = None + add_42 = output_40 + output_35 + output_40 = output_35 = None + output_41 = torch.nn.functional.layer_norm( + add_42, + (768,), + l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_42 = l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_6 = output_41[slice(0, None, None)] + detach_6 = new_mem_6.detach() + new_mem_6 = None + q_head_h_6 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_41, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_q_ = None + k_head_h_6 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_41, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_k_ = None + v_head_h_6 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_41, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_v_ = None + type_7 = pos_emb_4.type(torch.float32) + k_head_r_6 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_7, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_, + ) + type_7 = l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_ = None + add_43 = ( + q_head_h_6 + + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_ = None + ac_6 = torch.functional.einsum("ibnd,jbnd->bnij", add_43, k_head_h_6) + add_43 = k_head_h_6 = None + add_44 = ( + q_head_h_6 + + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_ = None + bd_6 = torch.functional.einsum("ibnd,jbnd->bnij", add_44, k_head_r_6) + add_44 = k_head_r_6 = None + x_24 = bd_6.reshape(1, 12, 26, 13) + bd_6 = None + x_25 = x_24[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_24 = None + x_26 = x_25.reshape(1, 12, 13, 25) + x_25 = None + arange_8 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_27 = torch.index_select(x_26, 3, arange_8) + x_26 = arange_8 = None + add_45 = ( + q_head_h_6 + + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_6 = ( + l_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_12 = torch.functional.einsum( + "ibnd,snd->ibns", + add_45, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_, + ) + add_45 = ( + l_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_13 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_12) + ef_12 = None + add_46 = ac_6 + x_27 + ac_6 = x_27 = None + add_47 = add_46 + ef_13 + add_46 = ef_13 = None + attn_score_12 = add_47 * 0.125 + add_47 = None + einsum_75 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_13 = 1e30 * einsum_75 + einsum_75 = None + attn_score_13 = attn_score_12 - mul_13 + attn_score_12 = mul_13 = None + attn_prob_12 = torch.nn.functional.softmax(attn_score_13, dim=3) + attn_score_13 = None + attn_prob_13 = torch.nn.functional.dropout(attn_prob_12, 0.1, False, False) + attn_prob_12 = None + attn_vec_6 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_13, v_head_h_6 + ) + attn_prob_13 = v_head_h_6 = None + attn_out_18 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_6, + l_self_modules_layer_modules_6_modules_rel_attn_parameters_o_, + ) + attn_vec_6 = ( + l_self_modules_layer_modules_6_modules_rel_attn_parameters_o_ + ) = None + attn_out_19 = torch.nn.functional.dropout(attn_out_18, 0.1, False, False) + attn_out_18 = None + attn_out_20 = attn_out_19 + output_41 + attn_out_19 = output_41 = None + output_42 = torch.nn.functional.layer_norm( + attn_out_20, + (768,), + l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_20 = l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_43 = torch._C._nn.linear( + output_42, + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_44 = torch._C._nn.gelu(output_43) + output_43 = None + output_45 = torch.nn.functional.dropout(output_44, 0.1, False, False) + output_44 = None + output_46 = torch._C._nn.linear( + output_45, + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_, + ) + output_45 = ( + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_47 = torch.nn.functional.dropout(output_46, 0.1, False, False) + output_46 = None + add_49 = output_47 + output_42 + output_47 = output_42 = None + output_48 = torch.nn.functional.layer_norm( + add_49, + (768,), + l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_49 = l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_7 = output_48[slice(0, None, None)] + detach_7 = new_mem_7.detach() + new_mem_7 = None + q_head_h_7 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_48, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_q_ = None + k_head_h_7 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_48, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_k_ = None + v_head_h_7 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_48, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_v_ = None + type_8 = pos_emb_4.type(torch.float32) + k_head_r_7 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_8, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_, + ) + type_8 = l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_ = None + add_50 = ( + q_head_h_7 + + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_ = None + ac_7 = torch.functional.einsum("ibnd,jbnd->bnij", add_50, k_head_h_7) + add_50 = k_head_h_7 = None + add_51 = ( + q_head_h_7 + + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_ = None + bd_7 = torch.functional.einsum("ibnd,jbnd->bnij", add_51, k_head_r_7) + add_51 = k_head_r_7 = None + x_28 = bd_7.reshape(1, 12, 26, 13) + bd_7 = None + x_29 = x_28[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_28 = None + x_30 = x_29.reshape(1, 12, 13, 25) + x_29 = None + arange_9 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_31 = torch.index_select(x_30, 3, arange_9) + x_30 = arange_9 = None + add_52 = ( + q_head_h_7 + + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_7 = ( + l_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_14 = torch.functional.einsum( + "ibnd,snd->ibns", + add_52, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_, + ) + add_52 = ( + l_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_15 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_14) + ef_14 = None + add_53 = ac_7 + x_31 + ac_7 = x_31 = None + add_54 = add_53 + ef_15 + add_53 = ef_15 = None + attn_score_14 = add_54 * 0.125 + add_54 = None + einsum_86 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_15 = 1e30 * einsum_86 + einsum_86 = None + attn_score_15 = attn_score_14 - mul_15 + attn_score_14 = mul_15 = None + attn_prob_14 = torch.nn.functional.softmax(attn_score_15, dim=3) + attn_score_15 = None + attn_prob_15 = torch.nn.functional.dropout(attn_prob_14, 0.1, False, False) + attn_prob_14 = None + attn_vec_7 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_15, v_head_h_7 + ) + attn_prob_15 = v_head_h_7 = None + attn_out_21 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_7, + l_self_modules_layer_modules_7_modules_rel_attn_parameters_o_, + ) + attn_vec_7 = ( + l_self_modules_layer_modules_7_modules_rel_attn_parameters_o_ + ) = None + attn_out_22 = torch.nn.functional.dropout(attn_out_21, 0.1, False, False) + attn_out_21 = None + attn_out_23 = attn_out_22 + output_48 + attn_out_22 = output_48 = None + output_49 = torch.nn.functional.layer_norm( + attn_out_23, + (768,), + l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_23 = l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_50 = torch._C._nn.linear( + output_49, + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_51 = torch._C._nn.gelu(output_50) + output_50 = None + output_52 = torch.nn.functional.dropout(output_51, 0.1, False, False) + output_51 = None + output_53 = torch._C._nn.linear( + output_52, + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_, + ) + output_52 = ( + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_54 = torch.nn.functional.dropout(output_53, 0.1, False, False) + output_53 = None + add_56 = output_54 + output_49 + output_54 = output_49 = None + output_55 = torch.nn.functional.layer_norm( + add_56, + (768,), + l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_56 = l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_8 = output_55[slice(0, None, None)] + detach_8 = new_mem_8.detach() + new_mem_8 = None + q_head_h_8 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_55, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_q_ = None + k_head_h_8 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_55, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_k_ = None + v_head_h_8 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_55, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_v_ = None + type_9 = pos_emb_4.type(torch.float32) + k_head_r_8 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_9, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_, + ) + type_9 = l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_ = None + add_57 = ( + q_head_h_8 + + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_ = None + ac_8 = torch.functional.einsum("ibnd,jbnd->bnij", add_57, k_head_h_8) + add_57 = k_head_h_8 = None + add_58 = ( + q_head_h_8 + + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_ = None + bd_8 = torch.functional.einsum("ibnd,jbnd->bnij", add_58, k_head_r_8) + add_58 = k_head_r_8 = None + x_32 = bd_8.reshape(1, 12, 26, 13) + bd_8 = None + x_33 = x_32[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_32 = None + x_34 = x_33.reshape(1, 12, 13, 25) + x_33 = None + arange_10 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_35 = torch.index_select(x_34, 3, arange_10) + x_34 = arange_10 = None + add_59 = ( + q_head_h_8 + + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_8 = ( + l_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_16 = torch.functional.einsum( + "ibnd,snd->ibns", + add_59, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_, + ) + add_59 = ( + l_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_17 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_16) + ef_16 = None + add_60 = ac_8 + x_35 + ac_8 = x_35 = None + add_61 = add_60 + ef_17 + add_60 = ef_17 = None + attn_score_16 = add_61 * 0.125 + add_61 = None + einsum_97 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_17 = 1e30 * einsum_97 + einsum_97 = None + attn_score_17 = attn_score_16 - mul_17 + attn_score_16 = mul_17 = None + attn_prob_16 = torch.nn.functional.softmax(attn_score_17, dim=3) + attn_score_17 = None + attn_prob_17 = torch.nn.functional.dropout(attn_prob_16, 0.1, False, False) + attn_prob_16 = None + attn_vec_8 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_17, v_head_h_8 + ) + attn_prob_17 = v_head_h_8 = None + attn_out_24 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_8, + l_self_modules_layer_modules_8_modules_rel_attn_parameters_o_, + ) + attn_vec_8 = ( + l_self_modules_layer_modules_8_modules_rel_attn_parameters_o_ + ) = None + attn_out_25 = torch.nn.functional.dropout(attn_out_24, 0.1, False, False) + attn_out_24 = None + attn_out_26 = attn_out_25 + output_55 + attn_out_25 = output_55 = None + output_56 = torch.nn.functional.layer_norm( + attn_out_26, + (768,), + l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_26 = l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_57 = torch._C._nn.linear( + output_56, + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_58 = torch._C._nn.gelu(output_57) + output_57 = None + output_59 = torch.nn.functional.dropout(output_58, 0.1, False, False) + output_58 = None + output_60 = torch._C._nn.linear( + output_59, + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_, + ) + output_59 = ( + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_61 = torch.nn.functional.dropout(output_60, 0.1, False, False) + output_60 = None + add_63 = output_61 + output_56 + output_61 = output_56 = None + output_62 = torch.nn.functional.layer_norm( + add_63, + (768,), + l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_63 = l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_9 = output_62[slice(0, None, None)] + detach_9 = new_mem_9.detach() + new_mem_9 = None + q_head_h_9 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_62, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_q_ = None + k_head_h_9 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_62, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_k_ = None + v_head_h_9 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_62, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_v_ = None + type_10 = pos_emb_4.type(torch.float32) + k_head_r_9 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_10, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_, + ) + type_10 = l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_ = None + add_64 = ( + q_head_h_9 + + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_ = None + ac_9 = torch.functional.einsum("ibnd,jbnd->bnij", add_64, k_head_h_9) + add_64 = k_head_h_9 = None + add_65 = ( + q_head_h_9 + + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_ = None + bd_9 = torch.functional.einsum("ibnd,jbnd->bnij", add_65, k_head_r_9) + add_65 = k_head_r_9 = None + x_36 = bd_9.reshape(1, 12, 26, 13) + bd_9 = None + x_37 = x_36[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_36 = None + x_38 = x_37.reshape(1, 12, 13, 25) + x_37 = None + arange_11 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_39 = torch.index_select(x_38, 3, arange_11) + x_38 = arange_11 = None + add_66 = ( + q_head_h_9 + + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_9 = ( + l_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_18 = torch.functional.einsum( + "ibnd,snd->ibns", + add_66, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_, + ) + add_66 = ( + l_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_19 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_18) + ef_18 = None + add_67 = ac_9 + x_39 + ac_9 = x_39 = None + add_68 = add_67 + ef_19 + add_67 = ef_19 = None + attn_score_18 = add_68 * 0.125 + add_68 = None + einsum_108 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_19 = 1e30 * einsum_108 + einsum_108 = None + attn_score_19 = attn_score_18 - mul_19 + attn_score_18 = mul_19 = None + attn_prob_18 = torch.nn.functional.softmax(attn_score_19, dim=3) + attn_score_19 = None + attn_prob_19 = torch.nn.functional.dropout(attn_prob_18, 0.1, False, False) + attn_prob_18 = None + attn_vec_9 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_19, v_head_h_9 + ) + attn_prob_19 = v_head_h_9 = None + attn_out_27 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_9, + l_self_modules_layer_modules_9_modules_rel_attn_parameters_o_, + ) + attn_vec_9 = ( + l_self_modules_layer_modules_9_modules_rel_attn_parameters_o_ + ) = None + attn_out_28 = torch.nn.functional.dropout(attn_out_27, 0.1, False, False) + attn_out_27 = None + attn_out_29 = attn_out_28 + output_62 + attn_out_28 = output_62 = None + output_63 = torch.nn.functional.layer_norm( + attn_out_29, + (768,), + l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_29 = l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_64 = torch._C._nn.linear( + output_63, + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_65 = torch._C._nn.gelu(output_64) + output_64 = None + output_66 = torch.nn.functional.dropout(output_65, 0.1, False, False) + output_65 = None + output_67 = torch._C._nn.linear( + output_66, + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_, + ) + output_66 = ( + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_ + ) = ( + l_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_68 = torch.nn.functional.dropout(output_67, 0.1, False, False) + output_67 = None + add_70 = output_68 + output_63 + output_68 = output_63 = None + output_69 = torch.nn.functional.layer_norm( + add_70, + (768,), + l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_70 = l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_10 = output_69[slice(0, None, None)] + detach_10 = new_mem_10.detach() + new_mem_10 = None + q_head_h_10 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_69, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_q_ = None + k_head_h_10 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_69, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_k_ = None + v_head_h_10 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_69, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_v_ = None + type_11 = pos_emb_4.type(torch.float32) + k_head_r_10 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_11, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_, + ) + type_11 = l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_ = None + add_71 = ( + q_head_h_10 + + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_ = None + ac_10 = torch.functional.einsum("ibnd,jbnd->bnij", add_71, k_head_h_10) + add_71 = k_head_h_10 = None + add_72 = ( + q_head_h_10 + + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_ = None + bd_10 = torch.functional.einsum("ibnd,jbnd->bnij", add_72, k_head_r_10) + add_72 = k_head_r_10 = None + x_40 = bd_10.reshape(1, 12, 26, 13) + bd_10 = None + x_41 = x_40[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_40 = None + x_42 = x_41.reshape(1, 12, 13, 25) + x_41 = None + arange_12 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_43 = torch.index_select(x_42, 3, arange_12) + x_42 = arange_12 = None + add_73 = ( + q_head_h_10 + + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_10 = ( + l_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_20 = torch.functional.einsum( + "ibnd,snd->ibns", + add_73, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_, + ) + add_73 = ( + l_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_21 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_20) + ef_20 = None + add_74 = ac_10 + x_43 + ac_10 = x_43 = None + add_75 = add_74 + ef_21 + add_74 = ef_21 = None + attn_score_20 = add_75 * 0.125 + add_75 = None + einsum_119 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + mul_21 = 1e30 * einsum_119 + einsum_119 = None + attn_score_21 = attn_score_20 - mul_21 + attn_score_20 = mul_21 = None + attn_prob_20 = torch.nn.functional.softmax(attn_score_21, dim=3) + attn_score_21 = None + attn_prob_21 = torch.nn.functional.dropout(attn_prob_20, 0.1, False, False) + attn_prob_20 = None + attn_vec_10 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_21, v_head_h_10 + ) + attn_prob_21 = v_head_h_10 = None + attn_out_30 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_10, + l_self_modules_layer_modules_10_modules_rel_attn_parameters_o_, + ) + attn_vec_10 = ( + l_self_modules_layer_modules_10_modules_rel_attn_parameters_o_ + ) = None + attn_out_31 = torch.nn.functional.dropout(attn_out_30, 0.1, False, False) + attn_out_30 = None + attn_out_32 = attn_out_31 + output_69 + attn_out_31 = output_69 = None + output_70 = torch.nn.functional.layer_norm( + attn_out_32, + (768,), + l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_32 = l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_71 = torch._C._nn.linear( + output_70, + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_72 = torch._C._nn.gelu(output_71) + output_71 = None + output_73 = torch.nn.functional.dropout(output_72, 0.1, False, False) + output_72 = None + output_74 = torch._C._nn.linear( + output_73, + l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_, + ) + output_73 = l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_ = ( + l_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_75 = torch.nn.functional.dropout(output_74, 0.1, False, False) + output_74 = None + add_77 = output_75 + output_70 + output_75 = output_70 = None + output_76 = torch.nn.functional.layer_norm( + add_77, + (768,), + l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_77 = l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_ = (None) + new_mem_11 = output_76[slice(0, None, None)] + detach_11 = new_mem_11.detach() + new_mem_11 = None + q_head_h_11 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_76, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_q_, + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_q_ = None + k_head_h_11 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_76, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_k_, + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_k_ = None + v_head_h_11 = torch.functional.einsum( + "ibh,hnd->ibnd", + output_76, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_v_, + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_v_ = None + type_12 = pos_emb_4.type(torch.float32) + pos_emb_4 = None + k_head_r_11 = torch.functional.einsum( + "ibh,hnd->ibnd", + type_12, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_, + ) + type_12 = l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_ = None + add_78 = ( + q_head_h_11 + + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_ = None + ac_11 = torch.functional.einsum("ibnd,jbnd->bnij", add_78, k_head_h_11) + add_78 = k_head_h_11 = None + add_79 = ( + q_head_h_11 + + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_ + ) + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_ = None + bd_11 = torch.functional.einsum("ibnd,jbnd->bnij", add_79, k_head_r_11) + add_79 = k_head_r_11 = None + x_44 = bd_11.reshape(1, 12, 26, 13) + bd_11 = None + x_45 = x_44[ + ( + slice(None, None, None), + slice(None, None, None), + slice(1, None, None), + slice(None, None, None), + ) + ] + x_44 = None + x_46 = x_45.reshape(1, 12, 13, 25) + x_45 = None + arange_13 = torch.arange( + 13, device=device(type="cuda", index=0), dtype=torch.int64 + ) + x_47 = torch.index_select(x_46, 3, arange_13) + x_46 = arange_13 = None + add_80 = ( + q_head_h_11 + + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_ + ) + q_head_h_11 = ( + l_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_ + ) = None + ef_22 = torch.functional.einsum( + "ibnd,snd->ibns", + add_80, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_, + ) + add_80 = ( + l_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_ + ) = None + ef_23 = torch.functional.einsum("ijbs,ibns->bnij", seg_mat_1, ef_22) + seg_mat_1 = ef_22 = None + add_81 = ac_11 + x_47 + ac_11 = x_47 = None + add_82 = add_81 + ef_23 + add_81 = ef_23 = None + attn_score_22 = add_82 * 0.125 + add_82 = None + einsum_130 = torch.functional.einsum("ijbn->bnij", non_tgt_mask_1) + non_tgt_mask_1 = None + mul_23 = 1e30 * einsum_130 + einsum_130 = None + attn_score_23 = attn_score_22 - mul_23 + attn_score_22 = mul_23 = None + attn_prob_22 = torch.nn.functional.softmax(attn_score_23, dim=3) + attn_score_23 = None + attn_prob_23 = torch.nn.functional.dropout(attn_prob_22, 0.1, False, False) + attn_prob_22 = None + attn_vec_11 = torch.functional.einsum( + "bnij,jbnd->ibnd", attn_prob_23, v_head_h_11 + ) + attn_prob_23 = v_head_h_11 = None + attn_out_33 = torch.functional.einsum( + "ibnd,hnd->ibh", + attn_vec_11, + l_self_modules_layer_modules_11_modules_rel_attn_parameters_o_, + ) + attn_vec_11 = ( + l_self_modules_layer_modules_11_modules_rel_attn_parameters_o_ + ) = None + attn_out_34 = torch.nn.functional.dropout(attn_out_33, 0.1, False, False) + attn_out_33 = None + attn_out_35 = attn_out_34 + output_76 + attn_out_34 = output_76 = None + output_77 = torch.nn.functional.layer_norm( + attn_out_35, + (768,), + l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_, + 1e-12, + ) + attn_out_35 = l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_ = (None) + output_78 = torch._C._nn.linear( + output_77, + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_, + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_, + ) + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_ = ( + l_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_ + ) = None + output_79 = torch._C._nn.gelu(output_78) + output_78 = None + output_80 = torch.nn.functional.dropout(output_79, 0.1, False, False) + output_79 = None + output_81 = torch._C._nn.linear( + output_80, + l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_, + l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_, + ) + output_80 = l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_ = ( + l_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_ + ) = None + output_82 = torch.nn.functional.dropout(output_81, 0.1, False, False) + output_81 = None + add_84 = output_82 + output_77 + output_82 = output_77 = None + output_83 = torch.nn.functional.layer_norm( + add_84, + (768,), + l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_, + l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_, + 1e-12, + ) + add_84 = l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_ = l_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_ = (None) + output_84 = torch.nn.functional.dropout(output_83, 0.1, False, False) + output_83 = None + permute = output_84.permute(1, 0, 2) + output_84 = None + output_85 = permute.contiguous() + permute = None + return ( + output_85, + detach, + detach_1, + detach_2, + detach_3, + detach_4, + detach_5, + detach_6, + detach_7, + detach_8, + detach_9, + detach_10, + detach_11, + ) diff --git a/samples/transformers-auto-model/xlnet-base-cased/weight_meta.py b/samples/transformers-auto-model/xlnet-base-cased/weight_meta.py new file mode 100644 index 0000000000..0c16061f37 --- /dev/null +++ b/samples/transformers-auto-model/xlnet-base-cased/weight_meta.py @@ -0,0 +1,2150 @@ +class Program_weight_tensor_meta_L_input_ids_: + name = "L_input_ids_" + shape = [1, 13] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [17, 30344, 10395, 27, 179, 22, 8771, 23228, 10500, 23, 9, 4, 3] + + +class Program_weight_tensor_meta_L_token_type_ids_: + name = "L_token_type_ids_" + shape = [1, 13] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] + + +class Program_weight_tensor_meta_L_attention_mask_: + name = "L_attention_mask_" + shape = [1, 13] + dtype = "torch.int64" + device = "cuda:0" + mean = None + std = None + data = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + + +class Program_weight_tensor_meta_L_self_modules_word_embedding_parameters_weight_: + name = "L_self_modules_word_embedding_parameters_weight_" + shape = [32000, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.212 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.270 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.009 + std = 0.186 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.010 + std = 0.195 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.330 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.963 + std = 0.120 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_0_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.173 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_0_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.113 + std = 0.084 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_0_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.065 + std = 0.131 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_0_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.013 + std = 0.149 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.045 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.014 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.199 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.313 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.315 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.234 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.942 + std = 0.178 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_1_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.010 + std = 0.219 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_1_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.104 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_1_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.068 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.929 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_1_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.188 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.192 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.378 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.022 + std = 0.297 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.233 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.921 + std = 0.238 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_2_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.212 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_2_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.102 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_2_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.083 + std = 0.138 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_2_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.155 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.014 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.385 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.433 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.352 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.217 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.905 + std = 0.201 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_3_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.021 + std = 0.526 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_3_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.105 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_3_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.082 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.118 + std = 0.102 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_3_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.024 + std = 0.317 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.149 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.233 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.240 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.204 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.907 + std = 0.153 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_4_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.005 + std = 0.226 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_4_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.080 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_4_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.105 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.097 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_4_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.041 + std = 0.149 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.015 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.004 + std = 0.154 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.213 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.018 + std = 0.224 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.237 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.924 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_5_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.163 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_5_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.081 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_5_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.365 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_5_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.058 + std = 0.257 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.013 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.007 + std = 0.134 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.003 + std = 0.134 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.214 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.107 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.073 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_6_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.174 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_6_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.073 + std = 0.094 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_6_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.114 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.360 + std = 0.053 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_6_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.049 + std = 0.191 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.097 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.143 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.002 + std = 0.207 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.131 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.034 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.118 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_7_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.018 + std = 0.140 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_7_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.073 + std = 0.101 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_7_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.477 + std = 0.055 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_7_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.036 + std = 0.239 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.037 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.012 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.092 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.012 + std = 0.133 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.157 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.117 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.036 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.407 + std = 0.071 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_8_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.014 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_8_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.070 + std = 0.080 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.031 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_8_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.115 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.402 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_8_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.027 + std = 0.221 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.041 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.048 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.011 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.116 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.125 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.004 + std = 0.183 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.103 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.035 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.358 + std = 0.062 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_9_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.011 + std = 0.090 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_9_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.086 + std = 0.076 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.032 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_9_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.101 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.536 + std = 0.069 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_9_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.057 + std = 0.263 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.033 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.040 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.042 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.010 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.009 + std = 0.176 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.162 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.006 + std = 0.175 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.099 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.038 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.372 + std = 0.058 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_10_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.020 + std = 0.124 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.001 + std = 0.043 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_10_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.067 + std = 0.077 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.029 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_10_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.001 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.329 + std = 0.178 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_10_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.033 + std = 0.205 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_q_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_q_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.039 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_k_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_k_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.050 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_v_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_v_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.052 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.008 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_w_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.156 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_r_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.006 + std = 0.157 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_r_s_bias_" + shape = [12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.007 + std = 0.128 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_seg_embed_" + shape = [2, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.075 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_parameters_o_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_parameters_o_" + shape = [768, 12, 64] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.049 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 1.584 + std = 0.144 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_: + name = "L_self_modules_layer_modules_11_modules_rel_attn_modules_layer_norm_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.066 + std = 0.209 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_: + name = ( + "L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_weight_" + ) + shape = [3072, 768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.000 + std = 0.044 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_: + name = "L_self_modules_layer_modules_11_modules_ff_modules_layer_1_parameters_bias_" + shape = [3072] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.046 + std = 0.120 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_: + name = ( + "L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_weight_" + ) + shape = [768, 3072] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.000 + std = 0.027 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_: + name = "L_self_modules_layer_modules_11_modules_ff_modules_layer_2_parameters_bias_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 0.002 + std = 0.085 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_: + name = "L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_weight_" + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = 2.417 + std = 0.397 + data = None + + +class Program_weight_tensor_meta_L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_: + name = ( + "L_self_modules_layer_modules_11_modules_ff_modules_layer_norm_parameters_bias_" + ) + shape = [768] + dtype = "torch.float32" + device = "cuda:0" + mean = -0.019 + std = 0.235 + data = None