forked from takezo5096/DNN
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathtest.cpp.rnn
149 lines (107 loc) · 4.27 KB
/
test.cpp.rnn
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#include <vector>
#include <iostream>
#include <chrono>
#include "function.h"
#include "variable.h"
#include "model.h"
#include "batchdata.h"
#include "autoencoder.h"
#include "optimizer_adam.h"
#include "optimizer_sgd_moment.h"
#include "word_embed.h"
using namespace std;
MallocCounter mallocCounter;
void toPVariable(PVariable x1, float *X){
x1->data.memSetHost(X);
}
void readSentences(WordEmbed &we){
std::ifstream ifs("ptb.train.txt");
std::string str;
if (ifs.fail())
{
std::cerr << "open failed" << std::endl;
}
while (getline(ifs, str))
{
we.add(str, false);
}
}
Model model;
PVariable forward_one_step(PVariable x, PVariable d, PVariable &s_h){
PVariable w_p = model.f("f_plus")->forward(model.f("w_xe")->forward(x), model.f("w_eh")->forward(s_h));
s_h = model.f("f_tanh")->forward(w_p);
PVariable s_y = model.f("w_hy")->forward(s_h);
PVariable loss = model.f("f_softmax_cross_entoropy")->forward(s_y, d);
return loss;
}
int main(){
cout << "create dataset" << endl;
WordEmbed we;
readSentences(we);
int batch_size = 20;
int epoch_size = 39;
//float dropout_p = 0.5;
int bprop_len = 10;
float learning_rate = 0.001;
int sample_size = we.getDataset().size();
int i_size = we.getVocabSize();
int n_size = 650;
int o_size = we.getVocabSize();
cout << "create model..." << endl;
Function *w_xe = new FunctionLinear(n_size, i_size);
Function *w_eh = new FunctionLinear(n_size, n_size);
Function *f_tanh = new FunctionTanh();
Function *f_plus = new FunctionPlus();
Function *w_hy = new FunctionLinear(o_size, n_size);
Function *f_softmax_cross_entoropy = new FunctionSoftmaxCrossEntropy();
Function *f_loss_plus = new FunctionPlus();
model.putF("w_xe", w_xe);
model.putF("w_eh", w_eh);
model.putF("f_tanh", f_tanh);
model.putF("f_plus", f_plus);
model.putF("w_hy", w_hy);
model.putF("f_softmax_cross_entoropy", f_softmax_cross_entoropy);
model.putF("f_loss_plus", f_loss_plus);
//OptimizerAdam optimizer(&model, learning_rate);
OptimizerSGDMoment optimizer(&model, learning_rate, 0.7);
optimizer.init();
vector<int> dataset = we.getDataset();
std::chrono::system_clock::time_point start, end;
cout << "epoch start" << endl;
for(int l=0; l<epoch_size; l++){
float cur_log_perp = 0;
PVariable loss_sum(new Variable(1, 1));
PVariable s_h(new Variable(n_size, batch_size));
start = std::chrono::system_clock::now();
for (int idx=0; idx<sample_size/batch_size-1; idx++){
BatchData bdata(we.getVocabSize(), we.getVocabSize(), batch_size);
we.toOneHots(bdata.X, idx*batch_size, idx*batch_size + batch_size);
we.toOneHots(bdata.D, idx*batch_size+1, idx*batch_size + batch_size+1);
PVariable x(new Variable(we.getVocabSize(), batch_size));
PVariable d(new Variable(we.getVocabSize(), batch_size));
toPVariable(x, bdata.X);
toPVariable(d, bdata.D);
PVariable loss = forward_one_step(x, d, s_h);
cur_log_perp += loss->val();
loss_sum = f_loss_plus->forward(loss_sum, loss);
if ((idx+1) % bprop_len == 0){
loss_sum->backward();
optimizer.update();
loss_sum->zero_grads();
loss_sum->unchain();
loss_sum->zeros();
}
if ((idx+1) % 100 == 0){
end = std::chrono::system_clock::now();
int elapsed = std::chrono::duration_cast<std::chrono::seconds>(end-start).count();
cout << "epoch:" << (l+1) << " idx:" << (idx+1) << "/" << sample_size/batch_size
//<< " perplexity:" << std::exp(cur_log_perp/100) << " iters/sec:" << ((float)100)/((float)elapsed) << endl;
<< " perplexity:" << cur_log_perp/100 << " iters/sec:" << ((float)100)/((float)elapsed) << endl;
start = std::chrono::system_clock::now();
cur_log_perp = 0;
}
} //for sample_size/batch_size end
optimizer.lr /= 1.2;
cout << "epoch:" << (l+1) << " change learning ratge:" << optimizer.lr << endl;
}//for epoch end
}