DataAugmentation/dataloader.py at main · somos-ubb/DataAugmentation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import numpy as np


class Gen_Data_loader():
    def __init__(self, batch_size, vocab_dict):
        self.batch_size = batch_size
        self.token_stream = []
        self.vocab_size = 0
        self.vocab_dict = vocab_dict

    def create_batches(self, data_file_list):
        """make self.token_stream into a integer stream."""
        self.token_stream = []
        print("load %s file data.." % ' '.join(data_file_list))
        for data_file in data_file_list:
            with open(data_file, 'r') as f:
                for line in f:
                    line = line.strip()
                    line = line.split()
                    parse_line = [int(x) for x in line]
                    self.token_stream.append(parse_line)

        self.num_batch = int(len(self.token_stream) / self.batch_size)
        # cut the taken_stream's length exactly equal to num_batch * batch_size
        self.token_stream = self.token_stream[:self.num_batch * self.batch_size]
        #############################################################################
        max_length = max(len(seq) for seq in self.token_stream)
        self.token_stream = [seq + [0] * (max_length - len(seq)) for seq in self.token_stream]
        #############################################################################
        self.sequence_batch = np.split(np.array(self.token_stream), max(1, self.num_batch), 0)
        self.pointer = 0
        print("      Load %d * %d batches" % (self.num_batch, self.batch_size))

    def next_batch(self):
        """take next batch by self.pointer"""
        ret = self.sequence_batch[self.pointer]
        self.pointer = (self.pointer + 1) % self.num_batch
        return ret

    def reset_pointer(self):
        self.pointer = 0


class Dis_Data_loader():
    def __init__(self, batch_size, vocab_dict, max_sequence_length):
        self.batch_size = batch_size
        self.sentences = np.array([])
        self.labels = np.array([])
        self.vocab_dict = vocab_dict
        self.max_sequence_length = max_sequence_length

    def load_train_data(self, positive_file_list, negative_file_list):
        # Load data
        positive_examples = []
        negative_examples = []
        for positive_file in positive_file_list:
            with open(positive_file)as fin:
                for line in fin:
                    line = line.strip()
                    line = line.split()
                    parse_line = [int(x) for x in line]
                    positive_examples.append(parse_line)
        for negative_file in negative_file_list:
            with open(negative_file)as fin:
                for line in fin:
                    line = line.strip()
                    line = line.split()
                    parse_line = [int(x) for x in line]
                    negative_examples.append(parse_line)

        self.sentences = np.array(positive_examples + negative_examples)
        #print(self.sentences.shape)
        #print(self.max_sequence_length)
        self.sentences = self.padding(self.sentences, self.max_sequence_length)

        # Generate labels
        positive_labels = [[0, 1] for _ in positive_examples]
        negative_labels = [[1, 0] for _ in negative_examples]
        self.labels = np.concatenate([positive_labels, negative_labels], 0)

        # Shuffle the data
        shuffle_indices = np.random.permutation(np.arange(len(self.labels)))
        self.sentences = self.sentences[shuffle_indices]
        self.labels = self.labels[shuffle_indices]

        # Split batches
        self.num_batch = int(len(self.labels) / self.batch_size)
        self.sentences = self.sentences[:self.num_batch * self.batch_size]
        self.labels = self.labels[:self.num_batch * self.batch_size]
        self.sentences_batches = np.split(self.sentences, self.num_batch, 0)
        self.labels_batches = np.split(self.labels, self.num_batch, 0)
        self.pointer = 0

    def next_batch(self):
        """take next batch (sentence, label) by self.pointer"""
        ret = self.sentences_batches[self.pointer], self.labels_batches[self.pointer]
        self.pointer = (self.pointer + 1) % self.num_batch
        return ret

    def reset_pointer(self):
        self.pointer = 0

    def padding(self, sentences, sequence_length):
        sequence_length = min(sequence_length, max(map(lambda x: len(x), sentences)))
        padded_sentences = []
        for i in range(len(sentences)):
            sentence = sentences[i]
            if len(sentence) < sequence_length:
                num_padding = sequence_length - len(sentence)
                new_sentence = np.concatenate((sentence, [0] * num_padding))
            else:
                new_sentence = sentence[:sequence_length]
            padded_sentences.append(new_sentence)
        return np.array(padded_sentences)
    #def padding(self, inputs, max_sequence_length):
    #    batch_size = len(inputs)
    #    inputs_batch_major = np.zeros(shape=[batch_size, max_sequence_length], dtype=np.int32)  # == PAD
    #    print(inputs_batch_major.shape)
    #    for i, seq in enumerate(inputs):
    #        for j, element in enumerate(seq):
    #            inputs_batch_major[i, j] = element
    #    return inputs_batch_major