Skip to content

Change from Quasi-Newton method to SVRG #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tndoan wants to merge 12 commits into
base: master
Choose a base branch
from
Open

Change from Quasi-Newton method to SVRG #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
323772c
fix warnings
tndoan Jul 7, 2015
d7e986a
add getter
tndoan Jul 25, 2015
23c5920
add getter of params
tndoan Jul 25, 2015
b54a830
add getters and constructor
tndoan Jul 26, 2015
8cfe30e
add getter to return the size of features
tndoan Jul 27, 2015
b6971b4
add learning rate as parameters of file
tndoan Jul 28, 2015
350c0b5
change from using Quasi-Newton technique from Stanford NLP package to…
tndoan Jul 28, 2015
0aaf493
add implementation of functions
tndoan Jul 28, 2015
3ed3362
add implementation of SVRG and SGD
tndoan Jul 28, 2015
2a6f477
organize code, add and use derivative at each data point for SVRG
tndoan Aug 7, 2015
58dfb33
change logli and derivatives to public to access
tndoan Aug 7, 2015
e31d792
extends LogliComputer to do parallel and get derivatives of each data…
tndoan Aug 7, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Applications/PuncConverter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public static void convert(String inFilename, String outFilename) throws Excepti
static DataSet readInFile(String filename, LabelMap labelmap) throws Exception {
BufferedReader in = new BufferedReader(new FileReader(filename));

ArrayList td = new ArrayList();
ArrayList<DataSequence> td = new ArrayList<>();
ArrayList<String> inps = new ArrayList<String>();
ArrayList<String> labels = new ArrayList<String>();
String line;
Expand Down
18 changes: 9 additions & 9 deletions src/Applications/PunctuationPredictor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public PunctuationPredictor(String filename) {
public DataSet readTagged(String filename) throws Exception {
BufferedReader in = new BufferedReader(new FileReader(filename));

ArrayList td = new ArrayList();
ArrayList<DataSequence> td = new ArrayList<>();
ArrayList<String> inps = new ArrayList<String>();
ArrayList<String> labels = new ArrayList<String>();
String line;
Expand Down Expand Up @@ -119,14 +119,14 @@ public void createFeatureGenerator() throws Exception {
/**
* Train the high-order semi-CRF.
*/
public void train() throws Exception {
public void train(String puncFilename) throws Exception {
// Set training file name and create output directory
String trainFilename = "punc.train";
File dir = new File("learntModels/");
dir.mkdirs();

// Read training data and save the label map
PuncConverter.convert("punc.tr", trainFilename);
PuncConverter.convert(puncFilename, trainFilename);
DataSet trainData = readTagged(trainFilename);
labelmap.write("learntModels/labelmap");

Expand All @@ -144,7 +144,7 @@ public void train() throws Exception {
/**
* Test the high-order semi-CRF.
*/
public void test() throws Exception {
public void test(String tsFilename) throws Exception {
// Read label map, features, and CRF model
labelmap.read("learntModels/labelmap");
createFeatureGenerator();
Expand All @@ -155,7 +155,7 @@ public void test() throws Exception {
// Run Viterbi algorithm
System.out.print("Running Viterbi...");
String testFilename = "punc.test";
PuncConverter.convert("punc.ts", testFilename);
PuncConverter.convert(tsFilename, testFilename);
DataSet testData = readTagged(testFilename);
long startTime = System.currentTimeMillis();
highOrderSemiCrfModel.runViterbi(testData.getSeqList());
Expand All @@ -181,12 +181,12 @@ public void test() throws Exception {
public static void main(String argv[]) throws Exception {
PunctuationPredictor puncPredictor = new PunctuationPredictor(argv[1]);
if (argv[0].toLowerCase().equals("all")) {
puncPredictor.train();
puncPredictor.test();
puncPredictor.train(argv[2]);
puncPredictor.test(argv[3]);
} else if (argv[0].toLowerCase().equals("train")) {
puncPredictor.train();
puncPredictor.train(argv[2]);
} else if (argv[0].toLowerCase().equals("test")) {
puncPredictor.test();
puncPredictor.test(argv[3]);
}
}
}
18 changes: 9 additions & 9 deletions src/Applications/ReferenceTagger.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public ReferenceTagger(String filename) {
public DataSet readTagged(String filename) throws IOException {
BufferedReader in = new BufferedReader(new FileReader(filename));

ArrayList td = new ArrayList();
ArrayList<DataSequence> td = new ArrayList<>();
ArrayList<String> inps = new ArrayList<String>();
ArrayList<String> labels = new ArrayList<String>();
String line;
Expand Down Expand Up @@ -118,9 +118,9 @@ public void createFeatureGenerator() throws Exception {
/**
* Train the high-order semi-CRF.
*/
public void train() throws Exception {
public void train(String trainFilename) throws Exception {
// Set training file name and create output directory
String trainFilename = "ref.train";
// String trainFilename = "ref.train";
File dir = new File("learntModels/");
dir.mkdirs();

Expand All @@ -142,7 +142,7 @@ public void train() throws Exception {
/**
* Test the high-order semi-CRF.
*/
public void test() throws Exception {
public void test(String testFilename) throws Exception {
// Read label map, features, and CRF model
labelmap.read("learntModels/labelmap");
createFeatureGenerator();
Expand All @@ -152,7 +152,7 @@ public void test() throws Exception {

// Run Viterbi algorithm
System.out.print("Running Viterbi...");
String testFilename = "ref.test";
// String testFilename = "ref.test";
DataSet testData = readTagged(testFilename);
long startTime = System.currentTimeMillis();
highOrderSemiCrfModel.runViterbi(testData.getSeqList());
Expand All @@ -178,12 +178,12 @@ public void test() throws Exception {
public static void main(String argv[]) throws Exception {
ReferenceTagger refTagger = new ReferenceTagger(argv[1]);
if (argv[0].toLowerCase().equals("all")) {
refTagger.train();
refTagger.test();
refTagger.train(argv[2]);
refTagger.test(argv[3]);
} else if (argv[0].toLowerCase().equals("train")) {
refTagger.train();
refTagger.train(argv[2]);
} else if (argv[0].toLowerCase().equals("test")) {
refTagger.test();
refTagger.test(argv[3]);
}
}
}
76 changes: 40 additions & 36 deletions src/HOCRF/FeatureGenerator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@

import java.io.*;
import java.util.*;

import Parallel.*;

/**
Expand All @@ -33,16 +34,16 @@ public class FeatureGenerator {
int maxOrder; // Maximum order of the CRF
Params params; // Parameters

HashMap obsMap; // Map from feature observation to its ID
HashMap patternMap; // Map from feature pattern to index
HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector
HashMap<String, Integer> obsMap; // Map from feature observation to its ID
HashMap<String, Integer> patternMap; // Map from feature pattern to index
HashMap<FeatureIndex, Integer> featureMap; // Map from FeatureIndex to its ID in lambda vector
ArrayList<Feature> featureList; // Map from feature ID to features

HashMap forwardStateMap; // Map from forward state to index
HashMap<String, Integer> forwardStateMap; // Map from forward state to index
ArrayList<Integer>[] forwardTransition1; // Map from piID to list of pkID (see paper)
ArrayList<Integer>[] forwardTransition2; // Map from piID to list of pkyID (see paper)

HashMap backwardStateMap; // Map from backward state to index
HashMap<String, Integer> backwardStateMap; // Map from backward state to index
int[][] backwardTransition; // Map from [siID,y] to skID (see paper)
ArrayList<Integer>[] allSuffixes; // Map from sID to its suffixes patID
ArrayList<String> backwardStateList; // List of backward states
Expand All @@ -66,7 +67,7 @@ public FeatureGenerator(ArrayList<FeatureType> fts, Params pr) {
* This method needs to be called before the training process.
* @param trainData List of training sequences
*/
public void initialize(ArrayList trainData) throws Exception {
public void initialize(ArrayList<DataSequence> trainData) throws Exception {
generateFeatureMap(trainData);
generateForwardStatesMap();
generateBackwardStatesMap();
Expand All @@ -85,7 +86,7 @@ public void write(String filename) throws Exception {

// Write observation map
out.println(obsMap.size());
Iterator iter = obsMap.keySet().iterator();
Iterator<String> iter = obsMap.keySet().iterator();
while (iter.hasNext()) {
String key = (String) iter.next();
out.println(key + " " + obsMap.get(key));
Expand All @@ -101,9 +102,9 @@ public void write(String filename) throws Exception {

// Write feature map
out.println(featureMap.size());
iter = featureMap.keySet().iterator();
while (iter.hasNext()) {
FeatureIndex fi = (FeatureIndex) iter.next();
Iterator<FeatureIndex> iter_fi = featureMap.keySet().iterator();
while (iter_fi.hasNext()) {
FeatureIndex fi = (FeatureIndex) iter_fi.next();
int index = (Integer) featureMap.get(fi);
Feature f = featureList.get(index);
out.println(f.obs + " " + f.pat + " " + f.value + " " + index);
Expand Down Expand Up @@ -139,7 +140,7 @@ public void read(String filename) throws Exception {

// Read observation map
int mapSize = Integer.parseInt(in.readLine());
obsMap = new HashMap();
obsMap = new HashMap<>();
for (int i = 0; i < mapSize; i++) {
String line = in.readLine();
StringTokenizer toks = new StringTokenizer(line);
Expand All @@ -150,7 +151,7 @@ public void read(String filename) throws Exception {

// Read pattern map
mapSize = Integer.parseInt(in.readLine());
patternMap = new HashMap();
patternMap = new HashMap<>();
for (int i = 0; i < mapSize; i++) {
String line = in.readLine();
StringTokenizer toks = new StringTokenizer(line);
Expand All @@ -161,7 +162,7 @@ public void read(String filename) throws Exception {

// Read feature map
mapSize = Integer.parseInt(in.readLine());
featureMap = new HashMap();
featureMap = new HashMap<>();
featureList = new ArrayList<Feature>(mapSize);
for (int i = 0; i < mapSize; i++) featureList.add(null);
for (int i = 0; i < mapSize; i++) {
Expand All @@ -178,7 +179,7 @@ public void read(String filename) throws Exception {

// Read forward state map
mapSize = Integer.parseInt(in.readLine());
forwardStateMap = new HashMap();
forwardStateMap = new HashMap<>();
forwardStateMap.put("", new Integer(0));
for (int i = 0; i < mapSize-1; i++) {
String line = in.readLine();
Expand All @@ -190,7 +191,7 @@ public void read(String filename) throws Exception {

// Read backward state map
mapSize = Integer.parseInt(in.readLine());
backwardStateMap = new HashMap();
backwardStateMap = new HashMap<>();
backwardStateList = new ArrayList<String>(mapSize);
for (int i = 0; i < mapSize; i++) backwardStateList.add(null);
for (int i = 0; i < mapSize; i++) {
Expand Down Expand Up @@ -278,7 +279,7 @@ public int getMaxOrder() {
* Generate the features for each training sequence.
* @param trainData List of training sequences
*/
public void generateSentenceFeat(ArrayList trainData) throws Exception {
public void generateSentenceFeat(ArrayList<DataSequence> trainData) throws Exception {
SentenceFeatGenerator gen = new SentenceFeatGenerator(trainData, this);
Scheduler sch = new Scheduler(gen, params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE);
sch.run();
Expand All @@ -288,10 +289,10 @@ public void generateSentenceFeat(ArrayList trainData) throws Exception {
* Generate the observation map, pattern map, feature map, and feature list from training data.
* @param trainData List of training sequences
*/
public void generateFeatureMap(ArrayList trainData) {
obsMap = new HashMap();
patternMap = new HashMap();
featureMap = new HashMap();
public void generateFeatureMap(ArrayList<DataSequence> trainData) {
obsMap = new HashMap<>();
patternMap = new HashMap<>();
featureMap = new HashMap<>();
featureList = new ArrayList<Feature>();
for (int t = 0; t < trainData.size(); t++) {
DataSequence seq = (DataSequence) trainData.get(t);
Expand Down Expand Up @@ -326,12 +327,12 @@ public void generateFeatureMap(ArrayList trainData) {
* Generate the forward state map.
*/
public void generateForwardStatesMap() {
forwardStateMap = new HashMap();
forwardStateMap = new HashMap<>();
forwardStateMap.put("", new Integer(0));
for (int i = 0; i < params.numLabels; i++) {
forwardStateMap.put("" + i, new Integer(forwardStateMap.size()));
}
Iterator iter = patternMap.keySet().iterator();
Iterator<String> iter = patternMap.keySet().iterator();
while (iter.hasNext()) {
String labelPat = (String) iter.next();
ArrayList<String> pats = Utility.generateProperPrefixes(labelPat);
Expand All @@ -347,12 +348,12 @@ public void generateForwardStatesMap() {
* Generate the backward state map and the backward state list.
*/
public void generateBackwardStatesMap() {
backwardStateMap = new HashMap();
backwardStateMap = new HashMap<>();
backwardStateList = new ArrayList<String>();
Iterator iter = forwardStateMap.keySet().iterator();
Iterator<String> iter = forwardStateMap.keySet().iterator();
while (iter.hasNext()) {
String p = (String) iter.next();
int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p));
// int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p));
for (int y = 0; y < params.numLabels; y++) {
String py = p.equals("") ? y + "" : y + "|" + p;
if (getBackwardStateIndex(py) == null) {
Expand All @@ -364,7 +365,7 @@ public void generateBackwardStatesMap() {
}

/**
* Generate the maximum posible pattern for a position.
* Generate the maximum possible pattern for a position.
* Note that patterns are in reversed order: y(t)|y(t-1)|y(t-2)|...
* @param seq Data sequence
* @param pos Input position
Expand Down Expand Up @@ -432,7 +433,7 @@ public ArrayList<String> generateObs(DataSequence seq, int pos) {
* @param map Map from strings to indices
* @return Index of the longest suffix of the input string from the input map.
*/
public Integer getLongestSuffixID(String p, HashMap map) {
public Integer getLongestSuffixID(String p, HashMap<String, Integer> map) {
ArrayList<String> suffixes = Utility.generateSuffixes(p);
for (int i = 0; i < suffixes.size(); i++) {
Integer index = (Integer) map.get(suffixes.get(i));
Expand All @@ -449,7 +450,7 @@ public Integer getLongestSuffixID(String p, HashMap map) {
* @param map Map from strings to indices
* @return The longest suffix of the input string from the input map.
*/
public String getLongestSuffix(String p, HashMap map) {
public String getLongestSuffix(String p, HashMap<String, Integer> map) {
ArrayList<String> suffixes = Utility.generateSuffixes(p);
for (int i = 0; i < suffixes.size(); i++) {
Integer index = (Integer) map.get(suffixes.get(i));
Expand All @@ -463,11 +464,12 @@ public String getLongestSuffix(String p, HashMap map) {
/**
* Build the information for the forward algorithm.
*/
public void buildForwardTransition() {
@SuppressWarnings("unchecked")
public void buildForwardTransition() {
forwardTransition1 = new ArrayList[forwardStateMap.size()];
forwardTransition2 = new ArrayList[forwardStateMap.size()];

Iterator iter = forwardStateMap.keySet().iterator();
Iterator<String> iter = forwardStateMap.keySet().iterator();
while (iter.hasNext()) {
String pk = (String) iter.next();
int pkID = getForwardStateIndex(pk);
Expand All @@ -488,15 +490,16 @@ public void buildForwardTransition() {
/**
* Build the information for the backward algorithm.
*/
public void buildBackwardTransition() {
@SuppressWarnings("unchecked")
public void buildBackwardTransition() {
backwardTransition = new int[backwardStateMap.size()][params.numLabels];
allSuffixes = new ArrayList[backwardStateMap.size()];

Iterator iter = backwardStateMap.keySet().iterator();
Iterator<String> iter = backwardStateMap.keySet().iterator();
while (iter.hasNext()) {
String si = (String) iter.next();
int siID = getBackwardStateIndex(si);
int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si));
// int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si));
for (int y = 0; y < params.numLabels; y++) {
String siy = y + "|" + si;
String sk = getLongestSuffix(siy, backwardStateMap);
Expand All @@ -517,14 +520,15 @@ public void buildBackwardTransition() {
/**
* Build the information to compute the marginals and expected feature scores.
*/
public void buildPatternTransition() {
@SuppressWarnings("unchecked")
public void buildPatternTransition() {
patternTransition1 = new ArrayList[patternMap.size()];
patternTransition2 = new ArrayList[patternMap.size()];

Iterator forwardIter = forwardStateMap.keySet().iterator();
Iterator<String> forwardIter = forwardStateMap.keySet().iterator();
while (forwardIter.hasNext()) {
String pi = (String) forwardIter.next();
int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi));
// int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi));
int piID = getForwardStateIndex(pi);
for (int y = 0; y < params.numLabels; y++) {
String piy = pi.equals("") ? y + "" : y + "|" + pi;
Expand Down
4 changes: 2 additions & 2 deletions src/HOCRF/Function.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
public class Function implements DiffFunction {

FeatureGenerator featureGen; // Feature generator
ArrayList trainData; // List of training sequences
ArrayList<DataSequence> trainData; // List of training sequences

// Private data structures to compute function value and derivatives
private Loglikelihood logli; // Loglikelihood values
Expand All @@ -41,7 +41,7 @@ public class Function implements DiffFunction {
* @param fgen Feature generator
* @param data Training data
*/
public Function(FeatureGenerator fgen, ArrayList data) {
public Function(FeatureGenerator fgen, ArrayList<DataSequence> data) {
featureGen = fgen;
trainData = data;
lambdaCache = null;
Expand Down
Loading