sentiment-classifier/test.py at master · tehmas/sentiment-classifier · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
# -*- coding: utf-8 -*-
import sys
import re
import numpy as np
import operator
import math

def read_model(filename):
    model_type = ''
    model = {}
    model['class_pr'] = {}
    curr_class = ''
    with open(filename, 'r') as f:
        for line in f:
            line = line.split(',')
            if line[0] == '#class':
                curr_class = line[1].replace('\n','')
                model[curr_class]={}
                model['class_pr'][curr_class] = line[2].replace('\n','')
            elif line[0] == '#model_type':
                model_type = line[1].replace('\n','')
            else:
                term = line[0]
                pr = float(line[1].replace('\n',''))
                model[curr_class][term]=pr
    f.close()
    return model, model_type

def multinomial_classify(model, test_filename):
    stop_words = np.loadtxt('stoplist.txt',dtype=str)
    f = open(test_filename)
    data = f.read()
    f.close()
    exp = r'\w+(\.?\w+)*'
    tokens = re.finditer(exp,data)
    terms = {}
    for token in tokens:
        if token:
            token = token.group()
            if token not in stop_words:
                token = token.replace('\n','')
                if terms.has_key(token):
                    terms[token] += 1
                else:
                    terms[token] = 1
    predictions = {}
    for c in model:
        if c != 'class_pr':
            predictions[c] = float(model['class_pr'][c])
            for term in terms:
                if model[c].has_key(term):
                    predictions[c] += float(model[c][term])
    pclass = max(predictions.iteritems(),key=operator.itemgetter(1))[0]
    return pclass, predictions

def binomial_classify(model, test_filename):
    stop_words = np.loadtxt('stoplist.txt',dtype=str)
    f = open(test_filename)
    data = f.read()
    f.close()
    exp = r'\w+(\.?\w+)*'
    tokens = re.finditer(exp,data)
    terms = {}
    for token in tokens:
        if token:
            token = token.group()
            if token not in stop_words:
                token = token.replace('\n','')
                if terms.has_key(token):
                    terms[token] += 1
                else:
                    terms[token] = 1
    predictions = {}
    for c in model:
        if c != 'class_pr':
            predictions[c] =  float(model['class_pr'][c])
            for term in terms:
                if model[c].has_key(term):
                    predictions[c] += math.log(float(model[c][term]),10)
                    del model[c][term]
            for term in model[c]:
                predictions[c] += math.log(float(1-model[c][term]))
    pclass = max(predictions.iteritems(),key=operator.itemgetter(1))[0]
    return pclass, predictions

def main(model_filename, test_filename):
    model, model_type = read_model(model_filename)
    if model_type == 'multinomial':
        pclass, predictions = multinomial_classify(model, test_filename)

    if model_type == 'binomial':
        pclass, predictions = binomial_classify(model, test_filename)

    print pclass
    return

if __name__=='__main__':
    if len(sys.argv) != 3:
        print "usage: python test.py <model_filename> <test_filename>"
    else:
        main(model_filename=sys.argv[1], test_filename=sys.argv[2])