sentiment-classifier/train.py at master · tehmas/sentiment-classifier · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
# -*- coding: utf-8 -*-

import numpy as np
import os
import sys
import re
import math

def calculate_class_pr(train_dir):
    class_dirs = os.listdir(train_dir)
    class_files = {}
    total_file_count = 0
    for directory in class_dirs:
        class_files[directory]= os.listdir(train_dir +'/'+ directory)
        total_file_count += len(class_files[directory])

    pr = {}
    for class_name, file_names in class_files.iteritems():
        pr[class_name] = math.log(float(len(file_names))/total_file_count,10)
    return pr

def calculate_term_binomial(train_dir):
    class_dirs = os.listdir(train_dir)
    class_files = {}
    exp = r'\w+(\.?\w+)*'
    terms = {}
    total_classes = len(train_dir)
    total_files = {}
    stop_words = np.loadtxt('stoplist.txt',dtype=str)
    for c in class_dirs:
        class_files[c] = os.listdir(train_dir+'/'+c)
        terms[c] = {}

        for file_name in class_files[c]:
            path = train_dir + '/' + c + '/' + file_name
            f = open(path)
            data = f.read()
            f.close()
            tokens = re.finditer(exp, data)
            f_dict = {}
            for token in tokens:
                if token:
                    token = token.group()
                    if token not in stop_words:
                        token = token.replace('\n','')
                        if f_dict.has_key(token) == False:
                            f_dict[token] = 1
                            if terms[c].has_key(token) == False:
                                terms[c][token] = 1
                            else:
                                terms[c][token] += 1
    pr = {}
    for c in terms:
        pr[c] = {}

    for c in terms:
        for token in terms[c]:
            pr[c][token] = (float(terms[c][token] + 1)/(len(class_files[c]) + len(terms)))
            for other_c in terms:
                if c != other_c and pr[other_c].has_key(token) == False:
                    pr[other_c][token] = (float(1)/(len(class_files[c]) + len(terms)))
    return pr

def calculate_term_multinomial(train_dir):
    class_dirs = os.listdir(train_dir)
    class_files = {}
    exp = r'\w+(\.?\w+)*'
    terms = {}
    vocab_count = 0
    for directory in class_dirs:
        class_files[directory]= os.listdir(train_dir +'/'+ directory)
        stop_words = np.loadtxt('stoplist.txt',dtype=str)
        terms[directory] = {}
        for file_name in class_files[directory]:
            path = train_dir + '/' + directory + '/' + file_name
            f = open(path)
            data = f.read()
            f.close()

            exp = r'\w+(\.?\w+)*'
            tokens = re.finditer(exp,data)
            for token in tokens:
                if token:
                    token = token.group()
                    if token not in stop_words:
                        token = token.replace('\n','')
                        if terms[directory].has_key(token):
                            terms[directory][token] += 1
                        else:
                            terms[directory][token] = 1
                        vocab_count += 1
    pr = {}

    for directory in terms:
        pr[directory] = {}

    for directory in terms:
        for token in terms[directory]:
            pr[directory][token] = math.log(float(terms[directory][token] + 1)/(vocab_count + len(terms[directory])),10)
            for other_dir in terms:
                if other_dir != directory and terms[other_dir].has_key(token) == False:
                    pr[other_dir][token]=math.log(float(0+1)/(vocab_count + len(terms[other_dir])),10)

    return pr
def main(train_dir,output_filename):
    class_pr = calculate_class_pr(train_dir)
    term_pr = calculate_term_multinomial(train_dir)
    f = open(output_filename + '_multinomial', 'w')
    f.write('#model_type,multinomial')
    f.write('\n')
    for class_name in class_pr:
        f.write('#class')
        f.write(',')
        f.write(class_name)
        f.write(',')
        f.write(str(class_pr[class_name]))
        f.write('\n')
        for token in term_pr[class_name]:
            f.write(token)
            f.write(',')
            f.write(str(term_pr[class_name][token]))
            f.write('\n')
    f.close()

    term_pr = calculate_term_binomial(train_dir)

    f = open(output_filename + '_binomial', 'w')
    f.write('#model_type,binomial')
    f.write('\n')
    for class_name in class_pr:
        f.write('#class')
        f.write(',')
        f.write(class_name)
        f.write(',')
        f.write(str(class_pr[class_name]))
        f.write('\n')
        for token in term_pr[class_name]:
            f.write(token)
            f.write(',')
            f.write(str(term_pr[class_name][token]))
            f.write('\n')
    f.close()

    return

if __name__=='__main__':
    #main('train', 'model1')
    if len(sys.argv) != 3:
        print "usage: python train.py <directory_name> <output_filename>"
    else:
        main(train_dir=sys.argv[1], output_filename=sys.argv[2])