New GELU test case

apache · Jan 25, 2025 · 043338b · 043338b
1 parent ba985bd
commit 043338b
Show file tree

Hide file tree

Showing 20 changed files with 145 additions and 1 deletion.
diff --git a/src/test/java/org/apache/sysds/test/applications/nn/transformers/BertLayerTest.java b/src/test/java/org/apache/sysds/test/applications/nn/transformers/BertLayerTest.java
@@ -36,11 +36,17 @@ public void setUp() {
 	}
 
 	@Test
-	public void testBertLayerForwardNormal() {
+	public void testBertLayerForwardNormalTanh() {
 		runBertLayerTest("test1", 5, 4, 6, 2, 3, 7, "tanh", 0, TEST_NAME_FORWARD, 
             1e-5, true);
 	}
 
+	@Test
+	public void testBertLayerForwardNormalGelu() {
+		runBertLayerTest("test2", 4, 4, 8, 2, 4, 7, "gelu", 0, TEST_NAME_FORWARD, 
+            1e-5, true);
+	}
+
 	private void runBertLayerTest(String testSuffix, int batchSize, int seqLength, int embeddingDim, int numHeads,
             int perHeadEmbeddingDim, int intermediateEmbeddingDim, String activation, int debug, String testname, double precision, 
             boolean isForward) {

diff --git a/src/test/resources/component/transformers/bert_layer/input_W_K_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_K_test2.csv
@@ -0,0 +1,8 @@
+0.014873,0.127848,-0.276551,-0.306393,0.044137,0.116741,0.004873,-0.350424
+0.227909,0.044769,-0.185308,0.175143,0.316675,0.265246,-0.060110,0.159592
+-0.267258,-0.002632,0.285492,-0.251829,0.216273,-0.113814,-0.186207,-0.169799
+-0.242719,-0.069891,-0.286925,-0.100361,-0.223521,0.000566,0.046730,-0.235940
+-0.205295,0.044359,-0.025387,-0.118623,0.158570,0.182018,0.292360,-0.203683
+0.247464,-0.080732,0.349749,-0.052357,-0.249925,-0.341919,-0.103351,0.203278
+-0.127090,-0.002484,0.127717,0.003867,-0.149845,0.255612,-0.209903,0.187233
+0.298218,0.045111,0.010010,0.291613,0.103988,-0.292361,-0.130758,0.271360
diff --git a/src/test/resources/component/transformers/bert_layer/input_W_Q_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_Q_test2.csv
@@ -0,0 +1,8 @@
+-0.123685,0.009826,-0.317605,-0.071714,-0.068100,-0.318219,-0.040798,0.169884
+-0.289780,-0.030501,-0.167612,0.193891,-0.069418,-0.023860,-0.157829,0.124861
+-0.075206,0.071553,0.240736,0.191146,-0.317261,0.310922,0.282720,-0.085021
+0.075574,0.224803,-0.002292,-0.340978,-0.305271,-0.144212,-0.285705,-0.074354
+-0.230328,0.334902,-0.175732,0.220540,-0.055324,0.319260,0.037938,-0.291357
+-0.018144,0.224526,-0.270932,-0.276659,0.004572,0.128041,-0.074023,0.191571
+0.253091,0.335668,-0.330874,-0.074745,-0.160610,-0.319068,0.252477,0.280714
+-0.036345,-0.025570,-0.298402,-0.143356,0.133183,0.223692,0.098693,0.241910
diff --git a/src/test/resources/component/transformers/bert_layer/input_W_V_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_V_test2.csv
@@ -0,0 +1,8 @@
+-0.334274,0.011264,-0.080849,0.244498,0.069431,0.122792,0.029533,0.165114
+0.076385,-0.237900,-0.015723,-0.263194,-0.262510,-0.129004,0.044147,-0.171997
+-0.198408,-0.285785,-0.215330,0.144839,0.058866,0.134202,-0.277945,-0.292986
+-0.315220,0.281811,0.119572,-0.118884,0.150589,0.235453,0.027785,-0.304028
+0.310023,0.057572,0.111782,-0.170578,0.139947,-0.184608,0.244825,0.352708
+-0.229602,0.293317,-0.007293,0.063514,-0.044505,0.003487,0.318592,0.224432
+-0.040221,-0.118525,-0.079515,-0.183656,-0.289839,0.146194,0.207801,-0.244388
+0.101291,0.104141,-0.217941,0.081460,-0.054502,0.027711,0.047377,0.138325
diff --git a/src/test/resources/component/transformers/bert_layer/input_W_context_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_context_test2.csv
@@ -0,0 +1,8 @@
+-0.212761,-0.063619,-0.239211,-0.241276,0.290790,0.116962,-0.199302,0.226094
+0.317851,0.137643,-0.348794,-0.057128,0.289475,0.353439,-0.198385,0.026281
+0.170822,0.062799,-0.283923,-0.229609,0.253087,0.183375,-0.272053,-0.166925
+0.192735,0.150426,0.279120,0.245506,0.272988,0.219786,0.237351,0.324932
+-0.221599,-0.120147,0.191285,-0.267289,0.314375,-0.123741,0.251352,0.144582
+0.101434,0.172383,0.331709,-0.172499,-0.090532,0.169645,-0.040239,-0.268398
+-0.123943,-0.246947,0.283239,-0.341565,0.155564,0.040626,-0.204596,0.338380
+0.276251,0.079852,-0.315739,-0.200728,0.314991,-0.084435,0.273263,0.268479
diff --git a/src/test/resources/component/transformers/bert_layer/input_W_intermediate_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_intermediate_test2.csv
@@ -0,0 +1,8 @@
+-0.095970,0.175377,0.300088,-0.339869,0.016123,0.019002,0.112700
+-0.068211,-0.323126,0.023189,-0.343534,-0.318806,0.167661,0.189790
+0.033835,-0.063266,-0.235639,-0.071723,0.293212,-0.283489,0.049253
+0.326977,-0.262741,-0.126673,0.237741,0.190369,-0.101691,-0.236557
+0.018929,-0.150856,0.077203,-0.334631,0.351431,-0.347146,-0.274117
+-0.218298,0.127383,-0.269520,0.293869,0.178619,-0.137706,-0.109077
+0.018121,-0.251069,0.175649,-0.141429,-0.233370,0.076272,0.155195
+0.169524,0.131425,-0.320980,0.103550,0.295070,-0.277597,0.348744
diff --git a/src/test/resources/component/transformers/bert_layer/input_W_out_test2.csv b/src/test/resources/component/transformers/bert_layer/input_W_out_test2.csv
@@ -0,0 +1,7 @@
+0.181837,0.233912,-0.264798,-0.350101,0.154676,0.095152,-0.091160,-0.001032
+-0.266817,0.209144,0.317217,0.266578,-0.111686,0.134062,0.132476,-0.183122
+-0.187210,0.012154,-0.041096,0.189420,0.126487,-0.184663,-0.273795,0.377318
+-0.311325,-0.116858,-0.316714,0.223724,-0.108745,0.033411,-0.222254,0.369147
+0.197237,-0.082181,-0.204501,0.319958,0.233681,0.219044,-0.191851,-0.285056
+-0.038513,0.050232,0.334430,-0.203704,-0.104868,-0.037606,0.347353,-0.306404
+0.290882,0.187361,0.345659,0.119352,-0.140904,0.115027,-0.101704,-0.286494
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_K_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_K_test2.csv
@@ -0,0 +1,8 @@
+0.128242
+-0.118073
+-0.098800
+0.104450
+0.290633
+0.096128
+-0.167282
+-0.166197
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_Q_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_Q_test2.csv
@@ -0,0 +1,8 @@
+-0.249389
+0.015768
+-0.249232
+-0.194626
+-0.206017
+0.120825
+-0.210687
+-0.007714
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_V_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_V_test2.csv
@@ -0,0 +1,8 @@
+0.266992
+0.353417
+0.309145
+0.273917
+-0.081055
+-0.124077
+0.290285
+0.198124
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_context_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_context_test2.csv
@@ -0,0 +1,8 @@
+-0.128867
+0.198752
+-0.200885
+-0.055407
+0.300203
+0.014609
+-0.250039
+-0.118169
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_intermediate_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_intermediate_test2.csv
@@ -0,0 +1,7 @@
+0.203303
+-0.039813
+0.123962
+-0.346858
+-0.301970
+0.164971
+-0.200258
diff --git a/src/test/resources/component/transformers/bert_layer/input_b_out_test2.csv b/src/test/resources/component/transformers/bert_layer/input_b_out_test2.csv
@@ -0,0 +1,8 @@
+-0.001823
+-0.096345
+-0.247395
+-0.135565
+0.071409
+-0.197483
+0.083750
+-0.086673
diff --git a/src/test/resources/component/transformers/bert_layer/input_beta_ln1_test2.csv b/src/test/resources/component/transformers/bert_layer/input_beta_ln1_test2.csv
@@ -0,0 +1,8 @@
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
diff --git a/src/test/resources/component/transformers/bert_layer/input_beta_ln2_test2.csv b/src/test/resources/component/transformers/bert_layer/input_beta_ln2_test2.csv
@@ -0,0 +1,8 @@
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
+0.000000
diff --git a/src/test/resources/component/transformers/bert_layer/input_gamma_ln1_test2.csv b/src/test/resources/component/transformers/bert_layer/input_gamma_ln1_test2.csv
@@ -0,0 +1,8 @@
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
diff --git a/src/test/resources/component/transformers/bert_layer/input_gamma_ln2_test2.csv b/src/test/resources/component/transformers/bert_layer/input_gamma_ln2_test2.csv
@@ -0,0 +1,8 @@
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
+1.000000
diff --git a/src/test/resources/component/transformers/bert_layer/input_states_test2.csv b/src/test/resources/component/transformers/bert_layer/input_states_test2.csv
@@ -0,0 +1,4 @@
+0.496257,0.768222,0.088477,0.132030,0.307423,0.634079,0.490093,0.896445,0.455628,0.632306,0.348893,0.401717,0.022326,0.168859,0.293888,0.518522,0.697668,0.800011,0.161029,0.282269,0.681609,0.915194,0.397100,0.874156,0.419408,0.552907,0.952738,0.036165,0.185231,0.373417,0.305100,0.932000
+0.175910,0.269834,0.150680,0.031720,0.208130,0.929799,0.723109,0.742336,0.526296,0.243658,0.584592,0.033153,0.138717,0.242235,0.815469,0.793161,0.278252,0.481959,0.819780,0.997067,0.698441,0.567546,0.835243,0.205599,0.593172,0.112347,0.153457,0.241708,0.726237,0.701080,0.203824,0.651054
+0.774486,0.436891,0.519091,0.615852,0.810188,0.980097,0.114688,0.316765,0.696505,0.914275,0.935104,0.941178,0.599507,0.065209,0.545996,0.187197,0.034023,0.944246,0.880180,0.001236,0.593586,0.415770,0.417719,0.271122,0.692278,0.203848,0.683296,0.752854,0.857936,0.686956,0.005132,0.175652
+0.749658,0.604651,0.109958,0.212090,0.970375,0.836909,0.281987,0.374158,0.023701,0.491013,0.123471,0.114322,0.472450,0.575073,0.295235,0.796689,0.195730,0.953685,0.842650,0.078359,0.375558,0.522561,0.572951,0.618587,0.696214,0.529950,0.256036,0.736594,0.020376,0.203647,0.374835,0.256443
diff --git a/src/test/resources/component/transformers/bert_layer/output_attention_test2.csv b/src/test/resources/component/transformers/bert_layer/output_attention_test2.csv
@@ -0,0 +1,4 @@
+0.235963,0.286636,0.247663,0.229737,0.238065,0.274944,0.247426,0.239566,0.232993,0.292855,0.246338,0.227815,0.238436,0.277112,0.245687,0.238766,0.263707,0.244682,0.250010,0.241600,0.260119,0.247386,0.250597,0.241898,0.263318,0.247697,0.249780,0.239204,0.262353,0.253734,0.254431,0.229482
+0.200993,0.247095,0.287640,0.264272,0.209195,0.248364,0.281387,0.261054,0.203293,0.250675,0.284920,0.261113,0.196908,0.246377,0.297310,0.259405,0.275405,0.250326,0.237604,0.236665,0.275410,0.243066,0.237995,0.243529,0.268634,0.238802,0.252534,0.240029,0.260009,0.251079,0.252050,0.236862
+0.233736,0.309380,0.193616,0.263268,0.234876,0.298143,0.208577,0.258404,0.233731,0.292249,0.219675,0.254344,0.236114,0.298053,0.203767,0.262066,0.252833,0.254184,0.246234,0.246750,0.267395,0.239596,0.231539,0.261470,0.250207,0.258886,0.242438,0.248469,0.253909,0.258431,0.234523,0.253137
+0.260840,0.206205,0.213076,0.319878,0.260534,0.215267,0.221396,0.302804,0.256224,0.219768,0.224008,0.300000,0.263226,0.215017,0.223917,0.297839,0.241882,0.252050,0.249805,0.256263,0.244105,0.250428,0.252672,0.252795,0.247877,0.255111,0.241012,0.256000,0.235736,0.265862,0.248117,0.250285
diff --git a/src/test/resources/component/transformers/bert_layer/output_states_test2.csv b/src/test/resources/component/transformers/bert_layer/output_states_test2.csv
@@ -0,0 +1,4 @@
+0.107880,1.019378,-1.493256,-1.389399,0.794790,0.638421,-0.760268,1.082454,0.502346,1.475414,-1.059900,-0.800042,0.679010,-0.141650,-1.574906,0.919728,0.197951,0.675718,-1.499572,-1.111723,1.262451,0.866343,-1.089658,0.698491,0.020563,0.650392,0.510717,-1.720302,0.648202,0.237416,-1.555443,1.208454
+-0.479673,0.021814,-1.219189,-1.467430,0.830685,1.627373,-0.183904,0.870324,0.499331,-0.030822,-0.247241,-2.259303,0.867174,0.090776,-0.247555,1.327639,-0.698871,-0.044917,-0.524429,0.206017,2.304515,0.538566,-0.838074,-0.942806,0.366909,-0.345850,-1.177876,-0.736904,1.781455,0.944009,-1.253939,0.422195
+0.448434,-0.028957,-0.718420,-0.310317,1.695408,1.122331,-1.657827,-0.550652,0.321436,1.200561,0.028985,0.019767,1.668159,-1.221590,-1.277247,-0.740072,-0.742072,1.290652,0.293041,-1.603472,1.570833,0.167195,-0.766135,-0.210041,0.479908,-0.365131,-0.354259,0.115244,1.883001,0.681843,-1.724637,-0.715968
+0.389834,0.344840,-1.333194,-0.933634,1.855805,0.720664,-1.056654,0.012339,-0.525639,0.504899,-1.274403,-1.064770,1.492330,0.777126,-0.923244,1.013701,-0.584799,1.352766,-0.084635,-1.859024,1.122414,0.390217,-0.837819,0.500880,1.150180,0.986524,-1.630600,0.097320,0.872525,-0.111189,-1.498034,0.133275
-Original file line number
+Diff line change
@@ -0,0 +1,8 @@
+.000000
+.000000
+.000000
+.000000
+.000000
+.000000
+.000000
+.000000