diff --git a/src/Applications/PuncConverter.java b/src/Applications/PuncConverter.java index 664acd1..85c08dc 100644 --- a/src/Applications/PuncConverter.java +++ b/src/Applications/PuncConverter.java @@ -49,7 +49,7 @@ public static void convert(String inFilename, String outFilename) throws Excepti static DataSet readInFile(String filename, LabelMap labelmap) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; diff --git a/src/Applications/PunctuationPredictor.java b/src/Applications/PunctuationPredictor.java index 72f13e4..4ae67f7 100644 --- a/src/Applications/PunctuationPredictor.java +++ b/src/Applications/PunctuationPredictor.java @@ -51,7 +51,7 @@ public PunctuationPredictor(String filename) { public DataSet readTagged(String filename) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; @@ -119,14 +119,14 @@ public void createFeatureGenerator() throws Exception { /** * Train the high-order semi-CRF. */ - public void train() throws Exception { + public void train(String puncFilename) throws Exception { // Set training file name and create output directory String trainFilename = "punc.train"; File dir = new File("learntModels/"); dir.mkdirs(); // Read training data and save the label map - PuncConverter.convert("punc.tr", trainFilename); + PuncConverter.convert(puncFilename, trainFilename); DataSet trainData = readTagged(trainFilename); labelmap.write("learntModels/labelmap"); @@ -144,7 +144,7 @@ public void train() throws Exception { /** * Test the high-order semi-CRF. */ - public void test() throws Exception { + public void test(String tsFilename) throws Exception { // Read label map, features, and CRF model labelmap.read("learntModels/labelmap"); createFeatureGenerator(); @@ -155,7 +155,7 @@ public void test() throws Exception { // Run Viterbi algorithm System.out.print("Running Viterbi..."); String testFilename = "punc.test"; - PuncConverter.convert("punc.ts", testFilename); + PuncConverter.convert(tsFilename, testFilename); DataSet testData = readTagged(testFilename); long startTime = System.currentTimeMillis(); highOrderSemiCrfModel.runViterbi(testData.getSeqList()); @@ -181,12 +181,12 @@ public void test() throws Exception { public static void main(String argv[]) throws Exception { PunctuationPredictor puncPredictor = new PunctuationPredictor(argv[1]); if (argv[0].toLowerCase().equals("all")) { - puncPredictor.train(); - puncPredictor.test(); + puncPredictor.train(argv[2]); + puncPredictor.test(argv[3]); } else if (argv[0].toLowerCase().equals("train")) { - puncPredictor.train(); + puncPredictor.train(argv[2]); } else if (argv[0].toLowerCase().equals("test")) { - puncPredictor.test(); + puncPredictor.test(argv[3]); } } } diff --git a/src/Applications/ReferenceTagger.java b/src/Applications/ReferenceTagger.java index acc5d43..3bd1ba6 100644 --- a/src/Applications/ReferenceTagger.java +++ b/src/Applications/ReferenceTagger.java @@ -51,7 +51,7 @@ public ReferenceTagger(String filename) { public DataSet readTagged(String filename) throws IOException { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; @@ -118,9 +118,9 @@ public void createFeatureGenerator() throws Exception { /** * Train the high-order semi-CRF. */ - public void train() throws Exception { + public void train(String trainFilename) throws Exception { // Set training file name and create output directory - String trainFilename = "ref.train"; +// String trainFilename = "ref.train"; File dir = new File("learntModels/"); dir.mkdirs(); @@ -142,7 +142,7 @@ public void train() throws Exception { /** * Test the high-order semi-CRF. */ - public void test() throws Exception { + public void test(String testFilename) throws Exception { // Read label map, features, and CRF model labelmap.read("learntModels/labelmap"); createFeatureGenerator(); @@ -152,7 +152,7 @@ public void test() throws Exception { // Run Viterbi algorithm System.out.print("Running Viterbi..."); - String testFilename = "ref.test"; +// String testFilename = "ref.test"; DataSet testData = readTagged(testFilename); long startTime = System.currentTimeMillis(); highOrderSemiCrfModel.runViterbi(testData.getSeqList()); @@ -178,12 +178,12 @@ public void test() throws Exception { public static void main(String argv[]) throws Exception { ReferenceTagger refTagger = new ReferenceTagger(argv[1]); if (argv[0].toLowerCase().equals("all")) { - refTagger.train(); - refTagger.test(); + refTagger.train(argv[2]); + refTagger.test(argv[3]); } else if (argv[0].toLowerCase().equals("train")) { - refTagger.train(); + refTagger.train(argv[2]); } else if (argv[0].toLowerCase().equals("test")) { - refTagger.test(); + refTagger.test(argv[3]); } } } diff --git a/src/HOCRF/FeatureGenerator.java b/src/HOCRF/FeatureGenerator.java index 3ddaa22..e0331de 100644 --- a/src/HOCRF/FeatureGenerator.java +++ b/src/HOCRF/FeatureGenerator.java @@ -21,6 +21,7 @@ import java.io.*; import java.util.*; + import Parallel.*; /** @@ -33,16 +34,16 @@ public class FeatureGenerator { int maxOrder; // Maximum order of the CRF Params params; // Parameters - HashMap obsMap; // Map from feature observation to its ID - HashMap patternMap; // Map from feature pattern to index - HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector + HashMap obsMap; // Map from feature observation to its ID + HashMap patternMap; // Map from feature pattern to index + HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector ArrayList featureList; // Map from feature ID to features - HashMap forwardStateMap; // Map from forward state to index + HashMap forwardStateMap; // Map from forward state to index ArrayList[] forwardTransition1; // Map from piID to list of pkID (see paper) ArrayList[] forwardTransition2; // Map from piID to list of pkyID (see paper) - HashMap backwardStateMap; // Map from backward state to index + HashMap backwardStateMap; // Map from backward state to index int[][] backwardTransition; // Map from [siID,y] to skID (see paper) ArrayList[] allSuffixes; // Map from sID to its suffixes patID ArrayList backwardStateList; // List of backward states @@ -66,7 +67,7 @@ public FeatureGenerator(ArrayList fts, Params pr) { * This method needs to be called before the training process. * @param trainData List of training sequences */ - public void initialize(ArrayList trainData) throws Exception { + public void initialize(ArrayList trainData) throws Exception { generateFeatureMap(trainData); generateForwardStatesMap(); generateBackwardStatesMap(); @@ -85,7 +86,7 @@ public void write(String filename) throws Exception { // Write observation map out.println(obsMap.size()); - Iterator iter = obsMap.keySet().iterator(); + Iterator iter = obsMap.keySet().iterator(); while (iter.hasNext()) { String key = (String) iter.next(); out.println(key + " " + obsMap.get(key)); @@ -101,9 +102,9 @@ public void write(String filename) throws Exception { // Write feature map out.println(featureMap.size()); - iter = featureMap.keySet().iterator(); - while (iter.hasNext()) { - FeatureIndex fi = (FeatureIndex) iter.next(); + Iterator iter_fi = featureMap.keySet().iterator(); + while (iter_fi.hasNext()) { + FeatureIndex fi = (FeatureIndex) iter_fi.next(); int index = (Integer) featureMap.get(fi); Feature f = featureList.get(index); out.println(f.obs + " " + f.pat + " " + f.value + " " + index); @@ -139,7 +140,7 @@ public void read(String filename) throws Exception { // Read observation map int mapSize = Integer.parseInt(in.readLine()); - obsMap = new HashMap(); + obsMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -150,7 +151,7 @@ public void read(String filename) throws Exception { // Read pattern map mapSize = Integer.parseInt(in.readLine()); - patternMap = new HashMap(); + patternMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -161,7 +162,7 @@ public void read(String filename) throws Exception { // Read feature map mapSize = Integer.parseInt(in.readLine()); - featureMap = new HashMap(); + featureMap = new HashMap<>(); featureList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) featureList.add(null); for (int i = 0; i < mapSize; i++) { @@ -178,7 +179,7 @@ public void read(String filename) throws Exception { // Read forward state map mapSize = Integer.parseInt(in.readLine()); - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < mapSize-1; i++) { String line = in.readLine(); @@ -190,7 +191,7 @@ public void read(String filename) throws Exception { // Read backward state map mapSize = Integer.parseInt(in.readLine()); - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) backwardStateList.add(null); for (int i = 0; i < mapSize; i++) { @@ -278,7 +279,7 @@ public int getMaxOrder() { * Generate the features for each training sequence. * @param trainData List of training sequences */ - public void generateSentenceFeat(ArrayList trainData) throws Exception { + public void generateSentenceFeat(ArrayList trainData) throws Exception { SentenceFeatGenerator gen = new SentenceFeatGenerator(trainData, this); Scheduler sch = new Scheduler(gen, params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); @@ -288,10 +289,10 @@ public void generateSentenceFeat(ArrayList trainData) throws Exception { * Generate the observation map, pattern map, feature map, and feature list from training data. * @param trainData List of training sequences */ - public void generateFeatureMap(ArrayList trainData) { - obsMap = new HashMap(); - patternMap = new HashMap(); - featureMap = new HashMap(); + public void generateFeatureMap(ArrayList trainData) { + obsMap = new HashMap<>(); + patternMap = new HashMap<>(); + featureMap = new HashMap<>(); featureList = new ArrayList(); for (int t = 0; t < trainData.size(); t++) { DataSequence seq = (DataSequence) trainData.get(t); @@ -326,12 +327,12 @@ public void generateFeatureMap(ArrayList trainData) { * Generate the forward state map. */ public void generateForwardStatesMap() { - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < params.numLabels; i++) { forwardStateMap.put("" + i, new Integer(forwardStateMap.size())); } - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { String labelPat = (String) iter.next(); ArrayList pats = Utility.generateProperPrefixes(labelPat); @@ -347,12 +348,12 @@ public void generateForwardStatesMap() { * Generate the backward state map and the backward state list. */ public void generateBackwardStatesMap() { - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(); - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String p = (String) iter.next(); - int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); +// int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); for (int y = 0; y < params.numLabels; y++) { String py = p.equals("") ? y + "" : y + "|" + p; if (getBackwardStateIndex(py) == null) { @@ -364,7 +365,7 @@ public void generateBackwardStatesMap() { } /** - * Generate the maximum posible pattern for a position. + * Generate the maximum possible pattern for a position. * Note that patterns are in reversed order: y(t)|y(t-1)|y(t-2)|... * @param seq Data sequence * @param pos Input position @@ -432,7 +433,7 @@ public ArrayList generateObs(DataSequence seq, int pos) { * @param map Map from strings to indices * @return Index of the longest suffix of the input string from the input map. */ - public Integer getLongestSuffixID(String p, HashMap map) { + public Integer getLongestSuffixID(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -449,7 +450,7 @@ public Integer getLongestSuffixID(String p, HashMap map) { * @param map Map from strings to indices * @return The longest suffix of the input string from the input map. */ - public String getLongestSuffix(String p, HashMap map) { + public String getLongestSuffix(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -463,11 +464,12 @@ public String getLongestSuffix(String p, HashMap map) { /** * Build the information for the forward algorithm. */ - public void buildForwardTransition() { + @SuppressWarnings("unchecked") + public void buildForwardTransition() { forwardTransition1 = new ArrayList[forwardStateMap.size()]; forwardTransition2 = new ArrayList[forwardStateMap.size()]; - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String pk = (String) iter.next(); int pkID = getForwardStateIndex(pk); @@ -488,15 +490,16 @@ public void buildForwardTransition() { /** * Build the information for the backward algorithm. */ - public void buildBackwardTransition() { + @SuppressWarnings("unchecked") + public void buildBackwardTransition() { backwardTransition = new int[backwardStateMap.size()][params.numLabels]; allSuffixes = new ArrayList[backwardStateMap.size()]; - Iterator iter = backwardStateMap.keySet().iterator(); + Iterator iter = backwardStateMap.keySet().iterator(); while (iter.hasNext()) { String si = (String) iter.next(); int siID = getBackwardStateIndex(si); - int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); +// int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); for (int y = 0; y < params.numLabels; y++) { String siy = y + "|" + si; String sk = getLongestSuffix(siy, backwardStateMap); @@ -517,14 +520,15 @@ public void buildBackwardTransition() { /** * Build the information to compute the marginals and expected feature scores. */ - public void buildPatternTransition() { + @SuppressWarnings("unchecked") + public void buildPatternTransition() { patternTransition1 = new ArrayList[patternMap.size()]; patternTransition2 = new ArrayList[patternMap.size()]; - Iterator forwardIter = forwardStateMap.keySet().iterator(); + Iterator forwardIter = forwardStateMap.keySet().iterator(); while (forwardIter.hasNext()) { String pi = (String) forwardIter.next(); - int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); +// int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); int piID = getForwardStateIndex(pi); for (int y = 0; y < params.numLabels; y++) { String piy = pi.equals("") ? y + "" : y + "|" + pi; diff --git a/src/HOCRF/Function.java b/src/HOCRF/Function.java index 291aafa..5661852 100644 --- a/src/HOCRF/Function.java +++ b/src/HOCRF/Function.java @@ -30,7 +30,7 @@ public class Function implements DiffFunction { FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences // Private data structures to compute function value and derivatives private Loglikelihood logli; // Loglikelihood values @@ -41,7 +41,7 @@ public class Function implements DiffFunction { * @param fgen Feature generator * @param data Training data */ - public Function(FeatureGenerator fgen, ArrayList data) { + public Function(FeatureGenerator fgen, ArrayList data) { featureGen = fgen; trainData = data; lambdaCache = null; diff --git a/src/HOCRF/HighOrderCRF.java b/src/HOCRF/HighOrderCRF.java index 072fe7b..7c7ac4f 100644 --- a/src/HOCRF/HighOrderCRF.java +++ b/src/HOCRF/HighOrderCRF.java @@ -47,7 +47,7 @@ public HighOrderCRF(FeatureGenerator fgen) { * Train a high-order CRF from data. * @param data Training data */ - public void train(ArrayList data) { + public void train(ArrayList data) { QNMinimizer qn = new QNMinimizer(); Function df = new Function(featureGen, data); lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); @@ -57,7 +57,7 @@ public void train(ArrayList data) { * Run Viterbi algorithm on testing data. * @param data Testing data */ - public void runViterbi(ArrayList data) throws Exception { + public void runViterbi(ArrayList data) throws Exception { Viterbi tester = new Viterbi(featureGen, lambda, data); Scheduler sch = new Scheduler(tester, featureGen.params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); diff --git a/src/HOCRF/LogliComputer.java b/src/HOCRF/LogliComputer.java index eefa2e3..f14e85f 100644 --- a/src/HOCRF/LogliComputer.java +++ b/src/HOCRF/LogliComputer.java @@ -30,7 +30,7 @@ public class LogliComputer implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences double[] lambda; // Lambda vector Loglikelihood logli; // Loglikelihood value and derivatives final int BASE = 1; // Base of the logAlpha array @@ -42,7 +42,7 @@ public class LogliComputer implements Schedulable { * @param td List of training sequences * @param loglh Initial loglikelihood and its derivatives (partially computed from class Function) */ - public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { + public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { curID = -1; featureGen = fgen; trainData = td; @@ -57,7 +57,7 @@ public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, */ public Object compute(int taskID) { Loglikelihood res = new Loglikelihood(lambda.length); - DataSequence seq = (DataSequence) trainData.get(taskID); + DataSequence seq = trainData.get(taskID); addFeatureScores(seq, res); double[][] logAlpha = computeLogAlpha(seq); diff --git a/src/HOCRF/Scorer.java b/src/HOCRF/Scorer.java index 3b6c47f..4ba31ba 100644 --- a/src/HOCRF/Scorer.java +++ b/src/HOCRF/Scorer.java @@ -19,9 +19,10 @@ package HOCRF; -import java.io.*; -import java.util.*; -import java.text.*; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; /** * Scorer class @@ -40,16 +41,16 @@ public class Scorer { * @param labelmap Label map * @param RM_SUFFIX If set to true, suffixes of labels after '-' will be removed */ - public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { + public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { labels = new String[trueData.size()][]; for (int i = 0; i < trueData.size(); i++) { - DataSequence seq = (DataSequence) trueData.get(i); + DataSequence seq = trueData.get(i); labels[i] = labelmap.revArray(seq.labels); } predicted = new String[predictedData.size()][]; for (int i = 0; i < predictedData.size(); i++) { - DataSequence seq = (DataSequence) predictedData.get(i); + DataSequence seq = predictedData.get(i); predicted[i] = labelmap.revArray(seq.labels); } diff --git a/src/HOCRF/SentenceFeatGenerator.java b/src/HOCRF/SentenceFeatGenerator.java index 482d89b..eca4119 100644 --- a/src/HOCRF/SentenceFeatGenerator.java +++ b/src/HOCRF/SentenceFeatGenerator.java @@ -20,6 +20,7 @@ package HOCRF; import java.util.*; + import Parallel.*; /** @@ -30,7 +31,7 @@ public class SentenceFeatGenerator implements Schedulable { int curID; // Current task ID (for parallelization) - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences FeatureGenerator featGen; // Feature generator /** @@ -38,7 +39,7 @@ public class SentenceFeatGenerator implements Schedulable { * @param data Training data * @param fgen Feature generator */ - public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { + public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { curID = -1; trainData = data; featGen = fgen; @@ -49,8 +50,9 @@ public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { * @param taskID Index of the training sequence * @return The updated sequence */ - public Object compute(int taskID) { - DataSequence seq = (DataSequence) trainData.get(taskID); + @SuppressWarnings("unchecked") + public Object compute(int taskID) { + DataSequence seq = trainData.get(taskID); seq.features = new ArrayList[seq.length()][featGen.patternMap.size()]; for (int pos = 0; pos < seq.length(); pos++) { diff --git a/src/HOCRF/Viterbi.java b/src/HOCRF/Viterbi.java index 71526b6..d622d40 100644 --- a/src/HOCRF/Viterbi.java +++ b/src/HOCRF/Viterbi.java @@ -32,7 +32,7 @@ public class Viterbi implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator double[] lambda; // Lambda vector - ArrayList data; // List of testing sequences + ArrayList data; // List of testing sequences final int BASE = 1; // Base of the logAlpha array /** @@ -41,7 +41,7 @@ public class Viterbi implements Schedulable { * @param lambda Lambda vector * @param data Testing data */ - public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { + public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { curID = -1; this.featureGen = featureGen; this.lambda = lambda; @@ -54,7 +54,7 @@ public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { * @return The updated sequence */ public Object compute(int taskID) { - DataSequence seq = (DataSequence) data.get(taskID); + DataSequence seq = data.get(taskID); double maxScore[][] = new double[seq.length() + 1][featureGen.forwardStateMap.size()]; String trace[][] = new String[seq.length()][featureGen.forwardStateMap.size()]; diff --git a/src/HOSemiCRF/DataSequence.java b/src/HOSemiCRF/DataSequence.java index 0ba49bb..569274c 100644 --- a/src/HOSemiCRF/DataSequence.java +++ b/src/HOSemiCRF/DataSequence.java @@ -19,8 +19,7 @@ package HOSemiCRF; -import java.io.*; -import java.util.*; +import java.io.BufferedWriter; /** * Class for a data sequence diff --git a/src/HOSemiCRF/ExtLogliComputer.java b/src/HOSemiCRF/ExtLogliComputer.java new file mode 100644 index 0000000..b5a0300 --- /dev/null +++ b/src/HOSemiCRF/ExtLogliComputer.java @@ -0,0 +1,46 @@ +package HOSemiCRF; + +import java.util.ArrayList; + +/** + * This class extends LogliComputer in order to do parallel and get the derivative of each data point. + * @author tndoan + * + */ +public class ExtLogliComputer extends LogliComputer { + + /** + * store derivative of each data points + */ + private double[][] eachDerivatives; + + public ExtLogliComputer(double[] lambdaValues, FeatureGenerator fgen, + ArrayList td, Loglikelihood loglh, double[][] eachDerivatives) { + super(lambdaValues, fgen, td, loglh); + this.eachDerivatives = eachDerivatives; + } + + /** + * Override compute method of super class to store each derivative to {@link ExtLogliComputer#eachDerivatives} + */ + @Override + public Object compute(int taskID) { + Object result = super.compute(taskID); + double[] d = ((Loglikelihood) result).derivatives; + + for(int i = 0; i < d.length; i++){ + eachDerivatives[taskID][i] += d[i]; + } + + return result; + } + + /** + * get the derivative of each data point + * @return 2d array result. result[i] is derivative of data point i. + */ + public double[][] getEachDerivative() { + return eachDerivatives; + } + +} \ No newline at end of file diff --git a/src/HOSemiCRF/FeatureGenerator.java b/src/HOSemiCRF/FeatureGenerator.java index 0e52000..595f481 100644 --- a/src/HOSemiCRF/FeatureGenerator.java +++ b/src/HOSemiCRF/FeatureGenerator.java @@ -21,6 +21,7 @@ import java.io.*; import java.util.*; + import Parallel.*; /** @@ -35,17 +36,17 @@ public class FeatureGenerator { Params params; // Parameters int[] maxMemory; // Maximum segment length for each label - HashMap obsMap; // Map from feature observation to its ID - HashMap patternMap; // Map from feature pattern to index - HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector + HashMap obsMap; // Map from feature observation to its ID + HashMap patternMap; // Map from feature pattern to index + HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector ArrayList featureList; // Map from feature ID to features - HashMap forwardStateMap; // Map from forward state to index + HashMap forwardStateMap; // Map from forward state to index ArrayList[] forwardTransition1; // Map from piID to list of pkID (see paper) ArrayList[] forwardTransition2; // Map from piID to list of pkyID (see paper) int[] lastForwardStateLabel; // Map from piID to its last label - HashMap backwardStateMap; // Map from backward state to index + HashMap backwardStateMap; // Map from backward state to index int[][] backwardTransition; // Map from [siID,y] to skID (see paper) ArrayList[] allSuffixes; // Map from sID to its suffixes patID ArrayList backwardStateList; @@ -71,7 +72,7 @@ public FeatureGenerator(ArrayList fts, Params pr) { * This method needs to be called before the training process. * @param trainData List of training sequences */ - public void initialize(ArrayList trainData) throws Exception { + public void initialize(ArrayList trainData) throws Exception { createMaxMemory(trainData); generateFeatureMap(trainData); generateForwardStatesMap(); @@ -91,7 +92,7 @@ public void write(String filename) throws Exception { // Write observation map out.println(obsMap.size()); - Iterator iter = obsMap.keySet().iterator(); + Iterator iter = obsMap.keySet().iterator(); while (iter.hasNext()) { String key = (String) iter.next(); out.println(key + " " + obsMap.get(key)); @@ -107,9 +108,9 @@ public void write(String filename) throws Exception { // Write feature map out.println(featureMap.size()); - iter = featureMap.keySet().iterator(); - while (iter.hasNext()) { - FeatureIndex fi = (FeatureIndex) iter.next(); + Iterator fIter = featureMap.keySet().iterator(); + while (fIter.hasNext()) { + FeatureIndex fi = (FeatureIndex) fIter.next(); int index = (Integer) featureMap.get(fi); Feature f = featureList.get(index); out.println(f.obs + " " + f.pat + " " + f.value + " " + index); @@ -150,7 +151,7 @@ public void read(String filename) throws Exception { // Read observation map int mapSize = Integer.parseInt(in.readLine()); - obsMap = new HashMap(); + obsMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -161,7 +162,7 @@ public void read(String filename) throws Exception { // Read pattern map mapSize = Integer.parseInt(in.readLine()); - patternMap = new HashMap(); + patternMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -172,7 +173,7 @@ public void read(String filename) throws Exception { // Read feature map mapSize = Integer.parseInt(in.readLine()); - featureMap = new HashMap(); + featureMap = new HashMap<>(); featureList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) featureList.add(null); for (int i = 0; i < mapSize; i++) { @@ -189,7 +190,7 @@ public void read(String filename) throws Exception { // Read forward state map mapSize = Integer.parseInt(in.readLine()); - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < mapSize-1; i++) { String line = in.readLine(); @@ -201,7 +202,7 @@ public void read(String filename) throws Exception { // Read backward state map mapSize = Integer.parseInt(in.readLine()); - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) backwardStateList.add(null); for (int i = 0; i < mapSize; i++) { @@ -295,7 +296,7 @@ public int getMaxOrder() { * Generate the observations for each training sequence. * @param trainData List of training sequences */ - public void generateSentenceObs(ArrayList trainData) throws Exception { + public void generateSentenceObs(ArrayList trainData) throws Exception { SentenceObsGenerator gen = new SentenceObsGenerator(trainData, this); Scheduler sch = new Scheduler(gen, params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); @@ -307,7 +308,7 @@ public void generateSentenceObs(ArrayList trainData) throws Exception { * Reset the segment information for each training sequence. * @param trainData List of training sequences */ - public void createMaxMemory(ArrayList trainData) throws Exception { + public void createMaxMemory(ArrayList trainData) throws Exception { maxMemory = new int[params.numLabels]; Arrays.fill(maxMemory, -1); @@ -342,10 +343,10 @@ public void createMaxMemory(ArrayList trainData) throws Exception { * Generate the observation map, pattern map, feature map, and feature list from training data. * @param trainData List of training sequences */ - public void generateFeatureMap(ArrayList trainData) { - obsMap = new HashMap(); - patternMap = new HashMap(); - featureMap = new HashMap(); + public void generateFeatureMap(ArrayList trainData) { + obsMap = new HashMap<>(); + patternMap = new HashMap<>(); + featureMap = new HashMap<>(); featureList = new ArrayList(); for (int t = 0; t < trainData.size(); t++) { DataSequence seq = (DataSequence) trainData.get(t); @@ -381,12 +382,12 @@ public void generateFeatureMap(ArrayList trainData) { * Generate the forward state map. */ public void generateForwardStatesMap() { - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < params.numLabels; i++) { forwardStateMap.put("" + i, new Integer(forwardStateMap.size())); } - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { String labelPat = (String) iter.next(); ArrayList pats = Utility.generateProperPrefixes(labelPat); @@ -402,9 +403,9 @@ public void generateForwardStatesMap() { * Generate the backward state map and the backward state list. */ public void generateBackwardStatesMap() { - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(); - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String p = (String) iter.next(); int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); @@ -499,7 +500,7 @@ public ArrayList generateObs(DataSequence seq, int segStart, int segEnd) * @param map Map from strings to indices * @return Index of the longest suffix of the input string from the input map. */ - public Integer getLongestSuffixID(String p, HashMap map) { + public Integer getLongestSuffixID(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -516,7 +517,7 @@ public Integer getLongestSuffixID(String p, HashMap map) { * @param map Map from strings to indices * @return The longest suffix of the input string from the input map. */ - public String getLongestSuffix(String p, HashMap map) { + public String getLongestSuffix(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -530,14 +531,15 @@ public String getLongestSuffix(String p, HashMap map) { /** * Build the information for the forward algorithm. */ - public void buildForwardTransition() { + @SuppressWarnings("unchecked") + public void buildForwardTransition() { forwardTransition1 = new ArrayList[forwardStateMap.size()]; forwardTransition2 = new ArrayList[forwardStateMap.size()]; lastForwardStateLabel = new int[forwardStateMap.size()]; - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { - String pk = (String) iter.next(); + String pk = iter.next(); int pkID = getForwardStateIndex(pk); lastForwardStateLabel[pkID] = pk.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pk)); @@ -559,13 +561,14 @@ public void buildForwardTransition() { /** * Build the information for the backward algorithm. */ - public void buildBackwardTransition() { + @SuppressWarnings("unchecked") + public void buildBackwardTransition() { backwardTransition = new int[backwardStateMap.size()][params.numLabels]; allSuffixes = new ArrayList[backwardStateMap.size()]; - Iterator iter = backwardStateMap.keySet().iterator(); + Iterator iter = backwardStateMap.keySet().iterator(); while (iter.hasNext()) { - String si = (String) iter.next(); + String si = iter.next(); int siID = getBackwardStateIndex(si); int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); for (int y = 0; y < params.numLabels; y++) { @@ -592,15 +595,16 @@ public void buildBackwardTransition() { /** * Build the information to compute the marginals and expected feature scores. */ - public void buildPatternTransition() { + @SuppressWarnings("unchecked") + public void buildPatternTransition() { patternTransition1 = new ArrayList[patternMap.size()]; patternTransition2 = new ArrayList[patternMap.size()]; lastPatternLabel = new int[patternMap.size()]; patternBackwardID = new int[patternMap.size()]; - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { - String p = (String) iter.next(); + String p = iter.next(); int pID = getPatternIndex(p).intValue(); patternBackwardID[pID] = getBackwardStateIndex(p); String lastY = Utility.getLastLabel(p); @@ -611,9 +615,9 @@ public void buildPatternTransition() { } } - Iterator forwardIter = forwardStateMap.keySet().iterator(); + Iterator forwardIter = forwardStateMap.keySet().iterator(); while (forwardIter.hasNext()) { - String pi = (String) forwardIter.next(); + String pi = forwardIter.next(); int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); int piID = getForwardStateIndex(pi); for (int y = 0; y < params.numLabels; y++) { @@ -719,4 +723,20 @@ public void printStatesStatistics() { } } } + + /** + * + * @return object which contains all value of parameters. + */ + public Params getParams() { + return params; + } + + /** + * + * @return the dimension of features + */ + public int getFeatureSize(){ + return featureMap.size(); + } } diff --git a/src/HOSemiCRF/Function.java b/src/HOSemiCRF/Function.java index 0a52bb7..19b1d1d 100644 --- a/src/HOSemiCRF/Function.java +++ b/src/HOSemiCRF/Function.java @@ -30,7 +30,7 @@ public class Function implements DiffFunction { FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences // Private data structures to compute function value and derivatives private Loglikelihood logli; // Loglikelihood values @@ -41,7 +41,7 @@ public class Function implements DiffFunction { * @param fgen Feature generator * @param data Training data */ - public Function(FeatureGenerator fgen, ArrayList data) { + public Function(FeatureGenerator fgen, ArrayList data) { featureGen = fgen; trainData = data; lambdaCache = null; diff --git a/src/HOSemiCRF/HighOrderSemiCRF.java b/src/HOSemiCRF/HighOrderSemiCRF.java index 2294770..9c04576 100644 --- a/src/HOSemiCRF/HighOrderSemiCRF.java +++ b/src/HOSemiCRF/HighOrderSemiCRF.java @@ -21,7 +21,9 @@ import java.io.*; import java.util.*; -import edu.stanford.nlp.optimization.*; + +import optimization.FirstOrderDiffFunction; +import optimization.SVRGMinimizer; import Parallel.*; /** @@ -47,17 +49,22 @@ public HighOrderSemiCRF(FeatureGenerator fgen) { * Train a high-order semi-CRF from data. * @param data Training data */ - public void train(ArrayList data) { - QNMinimizer qn = new QNMinimizer(); - Function df = new Function(featureGen, data); - lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); + public void train(ArrayList data) { + // use library to do minimization +// QNMinimizer qn = new QNMinimizer(); +// Function df = new Function(featureGen, data); +// lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); + + FirstOrderDiffFunction func = new FirstOrderDiffFunction(featureGen, data); + SVRGMinimizer svrg = new SVRGMinimizer(); + lambda = svrg.minimize(func, lambda, featureGen.params.getLearningRate(), featureGen.params.maxIters, featureGen.params.epsForConvergence); } /** * Run Viterbi algorithm on testing data. * @param data Testing data */ - public void runViterbi(ArrayList data) throws Exception { + public void runViterbi(ArrayList data) throws Exception { Viterbi tester = new Viterbi(featureGen, lambda, data); Scheduler sch = new Scheduler(tester, featureGen.params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); diff --git a/src/HOSemiCRF/LogliComputer.java b/src/HOSemiCRF/LogliComputer.java index 53946d7..fb71418 100644 --- a/src/HOSemiCRF/LogliComputer.java +++ b/src/HOSemiCRF/LogliComputer.java @@ -30,7 +30,7 @@ public class LogliComputer implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences double[] lambda; // Lambda vector Loglikelihood logli; // Loglikelihood value and derivatives final int BASE = 1; // Base of the logAlpha array @@ -42,7 +42,7 @@ public class LogliComputer implements Schedulable { * @param td List of training sequences * @param loglh Initial loglikelihood and its derivatives (partially computed from class Function) */ - public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { + public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { curID = -1; featureGen = fgen; trainData = td; diff --git a/src/HOSemiCRF/Loglikelihood.java b/src/HOSemiCRF/Loglikelihood.java index febf0c3..1e5e18e 100644 --- a/src/HOSemiCRF/Loglikelihood.java +++ b/src/HOSemiCRF/Loglikelihood.java @@ -22,11 +22,12 @@ /** * Loglikelihood class * @author Nguyen Viet Cuong + * @author tndoan */ public class Loglikelihood { - double logli; // Loglikelihood value - double derivatives[]; // Loglikelihood derivatives + public double logli; // Loglikelihood value + public double derivatives[]; // Loglikelihood derivatives /** * Construct a loglikelihood with a given number of features. @@ -36,4 +37,31 @@ public Loglikelihood(int n) { logli = 0; derivatives = new double[n]; } + + /** + * construct loglikelihood with initial loglikelihood and derivative + * @param logli initial loglikelihood + * @param dev initial derivative + */ + public Loglikelihood(double logli, double[] dev) { + this.logli = logli; + derivatives = new double[dev.length]; + System.arraycopy(dev, 0, derivatives, 0, dev.length); + } + + /** + * + * @return the log likelihood + */ + public double getLogli() { + return logli; + } + + /** + * + * @return the derivative + */ + public double[] getDerivatives() { + return derivatives; + } } diff --git a/src/HOSemiCRF/Params.java b/src/HOSemiCRF/Params.java index ac68320..1bf5632 100644 --- a/src/HOSemiCRF/Params.java +++ b/src/HOSemiCRF/Params.java @@ -25,6 +25,7 @@ /** * Parameters class * @author Nguyen Viet Cuong + * @author tndoan */ public class Params { @@ -34,6 +35,11 @@ public class Params { int maxSegment = -1; // Maximum segment length double invSigmaSquare = 1.0; // Inverse of Sigma Squared double epsForConvergence = 0.001; // Convergence Precision + + /** + * learning rate for first order minimizer + */ + double learningRate = 0.1; // /** * Construct a parameters object. @@ -59,6 +65,65 @@ public Params(String filename, int nl) throws IOException { if ((value = options.getProperty("epsForConvergence")) != null) { epsForConvergence = Double.parseDouble(value); } + if ((value = options.getProperty("learningRate")) != null){ + learningRate = Double.parseDouble(value); + } numLabels = nl; } + + /** + * + * @return + */ + public int getNumLabels() { + return numLabels; + } + + /** + * + * @return + */ + public int getMaxIters() { + return maxIters; + } + + /** + * + * @return + */ + public int getNumthreads() { + return numthreads; + } + + /** + * + * @return + */ + public int getMaxSegment() { + return maxSegment; + } + + /** + * + * @return + */ + public double getInvSigmaSquare() { + return invSigmaSquare; + } + + /** + * + * @return + */ + public double getEpsForConvergence() { + return epsForConvergence; + } + + /** + * + * @return the value of learning rate + */ + public double getLearningRate() { + return learningRate; + } } diff --git a/src/HOSemiCRF/Scorer.java b/src/HOSemiCRF/Scorer.java index 208ff33..c4f7fc1 100644 --- a/src/HOSemiCRF/Scorer.java +++ b/src/HOSemiCRF/Scorer.java @@ -19,9 +19,11 @@ package HOSemiCRF; -import java.io.*; -import java.util.*; -import java.text.*; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Hashtable; +import java.util.Vector; /** * Scorer class @@ -40,7 +42,7 @@ public class Scorer { * @param labelmap Label map * @param RM_SUFFIX If set to true, suffixes of labels after '-' will be removed */ - public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { + public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { labels = new String[trueData.size()][]; for (int i = 0; i < trueData.size(); i++) { DataSequence seq = (DataSequence) trueData.get(i); diff --git a/src/HOSemiCRF/SentenceObsGenerator.java b/src/HOSemiCRF/SentenceObsGenerator.java index d1128df..fb7c994 100644 --- a/src/HOSemiCRF/SentenceObsGenerator.java +++ b/src/HOSemiCRF/SentenceObsGenerator.java @@ -30,7 +30,7 @@ public class SentenceObsGenerator implements Schedulable { int curID; // Current task ID (for parallelization) - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences FeatureGenerator featGen; // Feature generator /** @@ -38,7 +38,7 @@ public class SentenceObsGenerator implements Schedulable { * @param data Training data * @param fgen Feature generator */ - public SentenceObsGenerator(ArrayList data, FeatureGenerator fgen) { + public SentenceObsGenerator(ArrayList data, FeatureGenerator fgen) { curID = -1; trainData = data; featGen = fgen; diff --git a/src/HOSemiCRF/Viterbi.java b/src/HOSemiCRF/Viterbi.java index 5ecd1fa..2b47958 100644 --- a/src/HOSemiCRF/Viterbi.java +++ b/src/HOSemiCRF/Viterbi.java @@ -32,7 +32,7 @@ public class Viterbi implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator double[] lambda; // Lambda vector - ArrayList data; // List of training sequences + ArrayList data; // List of training sequences final int BASE = 1; // Base of the logAlpha array /** @@ -41,7 +41,7 @@ public class Viterbi implements Schedulable { * @param lambda Lambda vector * @param data Training data */ - public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { + public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { curID = -1; this.featureGen = featureGen; this.lambda = lambda; diff --git a/src/OCR/OCR.java b/src/OCR/OCR.java index 8d1ae2f..081de57 100644 --- a/src/OCR/OCR.java +++ b/src/OCR/OCR.java @@ -44,7 +44,7 @@ public OCR(String filename, String fold) { public DataSet readTagged(String filename, int trainFold, boolean isTraining) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; diff --git a/src/Parallel/Scheduler.java b/src/Parallel/Scheduler.java index a8830ee..0f17bf8 100644 --- a/src/Parallel/Scheduler.java +++ b/src/Parallel/Scheduler.java @@ -99,7 +99,7 @@ class SimpleTask implements Schedulable { double[] ans; int curID; - static final int N = 1000000; +// static final int N = 1000000; int nTasks; public SimpleTask(int nTasks) { @@ -108,7 +108,8 @@ public SimpleTask(int nTasks) { ans = new double[2]; } - public void showResult() { + @SuppressWarnings("unused") + public void showResult() { System.out.println(ans[0] + " " + ans[1]); } @@ -129,7 +130,8 @@ public Object compute(int taskID) { return result; } - public int getNumCompletedTasks() { + @SuppressWarnings("unused") + public int getNumCompletedTasks() { return curID; } diff --git a/src/optimization/AbstractSVRGFunction.java b/src/optimization/AbstractSVRGFunction.java new file mode 100644 index 0000000..6a12cc8 --- /dev/null +++ b/src/optimization/AbstractSVRGFunction.java @@ -0,0 +1,54 @@ +package optimization; + + +/** + * It is the abstract of function + * @author tndoan + * + */ +public abstract class AbstractSVRGFunction { + // similar AbstractStochasticCachingDiffUpdateFunction + + /** + * + * @return the number of data points + */ + public abstract int getNumberOfDataPoints(); + + /** + * Get value of function + * @param w value of parameters + * @return value of function + */ + public abstract double valueAt(double[] w); + + /** + * take derivative of function of whole data + * @param w value of parameters + * @return array corresponding to the derivative + */ + public abstract double[] takeDerivative(double[] w); + + /** + * take derivative of function of data point whose indices are given + * @param w value of parameters + * @param index array which contains index of data point that we will use + * @return array of derivative + */ + public abstract double[] takeDerivative(double[] w, int[] index); + + /** + * take derivative of function of data point whose index is given + * @param w value of parameter + * @param index index of data point in array + * @return derivative of function with parameter w at data point whose index is specified + */ + public abstract double[] takeDerivative(double[] w, int index); + + /** + * take derivative of function of each data point + * @param w value of parameter + * @return 2d array derivative of function with parameter w at each data point. result[i] is the derivative of data point i. + */ + public abstract double[][] takeEachDerivative(double[] w); +} \ No newline at end of file diff --git a/src/optimization/FirstOrderDiffFunction.java b/src/optimization/FirstOrderDiffFunction.java new file mode 100644 index 0000000..8a06665 --- /dev/null +++ b/src/optimization/FirstOrderDiffFunction.java @@ -0,0 +1,114 @@ +package optimization; + +import java.util.ArrayList; + +import HOSemiCRF.DataSequence; +import HOSemiCRF.ExtLogliComputer; +import HOSemiCRF.FeatureGenerator; +import HOSemiCRF.LogliComputer; +import HOSemiCRF.Loglikelihood; +import Parallel.Scheduler; + +public class FirstOrderDiffFunction extends AbstractSVRGFunction { + + FeatureGenerator featureGen; // Feature generator + ArrayList data; + + // Stored results + private Loglikelihood logli; + private double[][] eachDerivatives; + + public FirstOrderDiffFunction(FeatureGenerator fg, ArrayList data) { + this.data = data; + this.featureGen = fg; + } + + /** + * Compute loglikelihood, derivatives for all data, and derivatives at each data point + * Store in eachDerivatives + * @param w value of parameters + */ + public void computeAllValues(double[] w) { + // init eachDerivative + logli = new Loglikelihood(w.length); + eachDerivatives = new double[data.size()][w.length]; + for (int i = 0; i < w.length; i++){ + for (int j = 0; j < data.size(); j++){ + eachDerivatives[j][i] = -(w[i] * featureGen.getParams().getInvSigmaSquare()); + } + logli.logli -= ((w[i] * w[i]) * featureGen.getParams().getInvSigmaSquare()) / 2; + logli.derivatives[i] = -(w[i] * featureGen.getParams().getInvSigmaSquare()); + } + + ExtLogliComputer logliComp = new ExtLogliComputer(w, featureGen, data, logli, eachDerivatives); + Scheduler sch = new Scheduler(logliComp, featureGen.getParams().getNumthreads(), Scheduler.DYNAMIC_NEXT_AVAILABLE); + try { + sch.run(); + } catch (Exception e) { + System.out.println("Errors occur when training in parallel! " + e); + } + + // Change sign to maximize + for (int i = 0; i < w.length; i++) { + logli.derivatives[i] = -logli.derivatives[i] / data.size(); + for (int j = 0; j < data.size(); j++){ + eachDerivatives[j][i] = -eachDerivatives[j][i] / data.size(); + } + } + logli.logli = -logli.logli / data.size(); + } + + /** + * Ensure that computeAllValues(w) was called before this + */ + @Override + public double valueAt(double[] w) { + return logli.getLogli(); + } + + @Override + // Ensure that computeAllValues(w) was called before this + public double[] takeDerivative(double[] w) { + return logli.getDerivatives(); + } + + @Override + // Ensure that computeAllValues(w) was called before this + public double[][] takeEachDerivative(double[] w){ + return eachDerivatives; + } + + @Override + public double[] takeDerivative(double[] w, int[] index) { + int l = w.length; + double[] result = new double[l]; + + for(int i : index){ + double[] eachDev = takeDerivative(w, i); + for (int j = 0; j < l; j++){ + result[j] += eachDev[j]; + } + } + + return result; + } + + @Override + public double[] takeDerivative(double[] w, int index) { + Loglikelihood llh = new Loglikelihood(w.length); // can set loglikelihood any value; we dont care about this value + LogliComputer llc = new LogliComputer(w, featureGen, data, llh); + double[] result = ((Loglikelihood) llc.compute(index)).getDerivatives(); + + for (int i = 0; i < w.length; i++) { + result[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); + result[i] = -result[i] / data.size(); // Change sign to maximize + } + + return result; + } + + @Override + public int getNumberOfDataPoints() { + return data.size(); + } +} \ No newline at end of file diff --git a/src/optimization/Minimizer.java b/src/optimization/Minimizer.java new file mode 100644 index 0000000..7f18ed7 --- /dev/null +++ b/src/optimization/Minimizer.java @@ -0,0 +1,67 @@ +package optimization; + +/** + * abstract class for implementing first-order optimization + * @author tndoan + * + */ +public abstract class Minimizer { + + /** + * isConv = false if iteration terminates because of running more than max iterations; otherwise, true + */ + protected boolean isConv; + + /** + * after running algorithm, isConv = true if algorithm terminates because of convergence; + * otherwise(reach maximum number of iteration) isConv = false + * @return isConv + */ + public boolean isConv() { + return isConv; + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init){ + return minimize(f, init, 0.1); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate){ + return minimize(f, init, learningRate, 50); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @param maxPasses maximum number of data pass + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate, int maxPasses){ + return minimize(f, init, learningRate, maxPasses, 0.001); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @param maxPasses maximum number of data pass + * @param funcTol threshold to stop iteration before reaching max number of iteration + * @return value that minimize function + */ + public abstract double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate, int maxPasses, double funcTol); +} \ No newline at end of file diff --git a/src/optimization/SGDMinimizer.java b/src/optimization/SGDMinimizer.java new file mode 100644 index 0000000..11b1cae --- /dev/null +++ b/src/optimization/SGDMinimizer.java @@ -0,0 +1,40 @@ +package optimization; + +/** + * implementation of Stochastic Gradient Descent + * @author tndoan + * + */ +public class SGDMinimizer extends Minimizer { + + @Override + public double[] minimize(AbstractSVRGFunction f, double[] init, + double learningRate, int maxPasses, double funcTol) { + + double[] result = new double[init.length]; + System.arraycopy(init, 0, result, 0, init.length); + double pre_obj = Double.MAX_VALUE; + + for (int i = 0; i < maxPasses; i++){ + for (int j = 0; j < f.getNumberOfDataPoints(); j++){ + + double[] dev = f.takeDerivative(result, j); + for (int k = 0; k < dev.length; k++){ + result[k] -= learningRate * dev[k]; + } +// System.out.println(f.valueAt(result)); + } + + // check convergence + double cur_obj = f.valueAt(result); + if (i > 1 && Math.abs(pre_obj - cur_obj) < funcTol) { + this.isConv = true; + break; + } + pre_obj = cur_obj; + System.out.println(f.valueAt(result)); + } + + return result; + } +} diff --git a/src/optimization/SVRGMinimizer.java b/src/optimization/SVRGMinimizer.java new file mode 100644 index 0000000..ce90b88 --- /dev/null +++ b/src/optimization/SVRGMinimizer.java @@ -0,0 +1,78 @@ +package optimization; + +import java.util.Random; + +/** + * It is the implementation of SVRG algorithm + * http://stat.rutgers.edu/home/tzhang/papers/nips13-svrg.pdf + * @author tndoan + * + */ +public class SVRGMinimizer extends Minimizer { + + @Override + public double[] minimize(AbstractSVRGFunction f, double[] init, + double learningRate, int maxPasses, double funcTol) { + int n = f.getNumberOfDataPoints(); + int upFreq = 10; + int d = init.length; + + double[] w_0_tilde = new double[d]; + double[] mu_tilde; // = new double[d]; + double[] w_tilde = new double[d]; + double[] w_0 = new double[d]; + + Random rand = new Random(123456987); + + // copy init value + System.arraycopy(init, 0, w_0_tilde, 0, d); + + ((FirstOrderDiffFunction)f).computeAllValues(w_0_tilde); + double pre_obj = f.valueAt(w_0_tilde); + double[][] eachDerivatives = f.takeEachDerivative(w_0_tilde); + mu_tilde = f.takeDerivative(w_0_tilde); + + for (int s = 0; s < maxPasses; s++){ + // w_tilde = w_0_tilde + System.arraycopy(w_0_tilde, 0, w_tilde, 0, d); + + // mu_tilde = (\sum_{i=1}^n \delta \psi_i (w_tilde)) / n + for (int i = 0; i < d; i++){ + mu_tilde[i] /= (double) n; + } + + //w_0 = w_tilde + System.arraycopy(w_tilde, 0, w_0, 0, d); + + for (int t = 0; t < upFreq; t++){ + int i_t = rand.nextInt(n); + double[] f1 = f.takeDerivative(w_0, i_t); + double[] f2 = eachDerivatives[i_t]; + + for (int i = 0; i < d; i++){ + w_0[i] = w_0[i] - learningRate * (f1[i] - f2[i] + mu_tilde[i]); + } + } + + // w_0_tilde = w_0 + System.arraycopy(w_0, 0, w_0_tilde, 0, d); + + // check convergence + ((FirstOrderDiffFunction)f).computeAllValues(w_0_tilde); + double curr_obj = f.valueAt(w_0_tilde); + mu_tilde = f.takeDerivative(w_0_tilde); + eachDerivatives = f.takeEachDerivative(w_0_tilde); + + double diff = Math.abs(curr_obj - pre_obj); + if (s > 1 && diff < funcTol) { + // s > 1 to ensure that objective function is calculated at least 1 time. + this.isConv = true; + break; + } + System.out.println("Objective function pre:" + pre_obj + " curr:" + curr_obj + " funcTol:" + funcTol); + pre_obj = curr_obj; + } + + return w_0_tilde; + } +} diff --git a/src/optimization/SquareLossFunction.java b/src/optimization/SquareLossFunction.java new file mode 100644 index 0000000..bc91a52 --- /dev/null +++ b/src/optimization/SquareLossFunction.java @@ -0,0 +1,107 @@ +package optimization; + +import java.util.ArrayList; + +/** + * implementation of square loss function. It is used to test optimization function + * P = \sum_{i=1}^n (w * x_i - y_i) ^ 2 / n + * where n is total number of data point + * @author tndoan + * + */ +public class SquareLossFunction extends AbstractSVRGFunction{ + ArrayList data; // data[i] = x_i + ArrayList response; // response[i] = y_i + + public SquareLossFunction(ArrayList array, double[] r) { + assert(array.size() == r.length); + + // init data + data = new ArrayList<>(); + for (double[] x : array){ + double[] d = new double[x.length]; + System.arraycopy(x, 0, d, 0, x.length); + data.add(d); + } + + // init response + response = new ArrayList<>(); + for (double y : r) + response.add(y); + } + + @Override + public int getNumberOfDataPoints(){ + return data.size(); + } + + @Override + public double valueAt(double[] w) { + int n = data.size(); // number of data points + double result = 0; + + for (int i = 0; i < n; i++){ + double[] point = data.get(i); + double y = response.get(i); + double phi = 0; // phi = w * point; w and point are vectors + for ( int j = 0; j < w.length; j++){ + phi += w[j] * point[j]; + } + + phi -= y; + result += phi * phi; + } + + result /= n; + + return result; + } + + @Override + public double[] takeDerivative(double[] w) { + double[] result = new double[w.length]; + int n = data.size(); // number of data points + + for (int i = 0; i < n; i++){ + double[] r = takeDerivative(w, i); + for (int j = 0; j < result.length; j++){ + result[j] += r[j]; + } + } + + return result; + } + + @Override + public double[] takeDerivative(double[] w, int[] index) { + // TODO Auto-generated method stub + return null; + } + + @Override + public double[] takeDerivative(double[] w, int index) { + double[] point = data.get(index); + double r = response.get(index); + + double[] result = new double[w.length]; + + double phi = 0.0; + for (int i = 0; i < w.length; i++){ + phi += w[i] * point[i]; + } + + phi -= r; + + for (int i = 0; i < w.length; i++){ + result[i] = phi * point[i] * 2; + } + + return result; + } + + @Override + public double[][] takeEachDerivative(double[] w) { + // TODO Auto-generated method stub + return null; + } +}