From 323772cdbe613c9d3bc40860dfcc605dc66f51b0 Mon Sep 17 00:00:00 2001 From: tndoan Date: Tue, 7 Jul 2015 14:19:12 +0800 Subject: [PATCH 01/12] fix warnings --- src/Applications/PuncConverter.java | 2 +- src/Applications/PunctuationPredictor.java | 18 ++--- src/Applications/ReferenceTagger.java | 18 ++--- src/HOCRF/FeatureGenerator.java | 76 ++++++++++---------- src/HOCRF/Function.java | 4 +- src/HOCRF/HighOrderCRF.java | 4 +- src/HOCRF/LogliComputer.java | 6 +- src/HOCRF/Scorer.java | 13 ++-- src/HOCRF/SentenceFeatGenerator.java | 10 +-- src/HOCRF/Viterbi.java | 6 +- src/HOSemiCRF/DataSequence.java | 3 +- src/HOSemiCRF/FeatureGenerator.java | 80 ++++++++++++---------- src/HOSemiCRF/Function.java | 4 +- src/HOSemiCRF/HighOrderSemiCRF.java | 4 +- src/HOSemiCRF/LogliComputer.java | 4 +- src/HOSemiCRF/Scorer.java | 10 +-- src/HOSemiCRF/SentenceObsGenerator.java | 4 +- src/HOSemiCRF/Viterbi.java | 4 +- src/OCR/OCR.java | 2 +- src/Parallel/Scheduler.java | 8 ++- 20 files changed, 147 insertions(+), 133 deletions(-) diff --git a/src/Applications/PuncConverter.java b/src/Applications/PuncConverter.java index 664acd1..85c08dc 100644 --- a/src/Applications/PuncConverter.java +++ b/src/Applications/PuncConverter.java @@ -49,7 +49,7 @@ public static void convert(String inFilename, String outFilename) throws Excepti static DataSet readInFile(String filename, LabelMap labelmap) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; diff --git a/src/Applications/PunctuationPredictor.java b/src/Applications/PunctuationPredictor.java index 72f13e4..4ae67f7 100644 --- a/src/Applications/PunctuationPredictor.java +++ b/src/Applications/PunctuationPredictor.java @@ -51,7 +51,7 @@ public PunctuationPredictor(String filename) { public DataSet readTagged(String filename) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; @@ -119,14 +119,14 @@ public void createFeatureGenerator() throws Exception { /** * Train the high-order semi-CRF. */ - public void train() throws Exception { + public void train(String puncFilename) throws Exception { // Set training file name and create output directory String trainFilename = "punc.train"; File dir = new File("learntModels/"); dir.mkdirs(); // Read training data and save the label map - PuncConverter.convert("punc.tr", trainFilename); + PuncConverter.convert(puncFilename, trainFilename); DataSet trainData = readTagged(trainFilename); labelmap.write("learntModels/labelmap"); @@ -144,7 +144,7 @@ public void train() throws Exception { /** * Test the high-order semi-CRF. */ - public void test() throws Exception { + public void test(String tsFilename) throws Exception { // Read label map, features, and CRF model labelmap.read("learntModels/labelmap"); createFeatureGenerator(); @@ -155,7 +155,7 @@ public void test() throws Exception { // Run Viterbi algorithm System.out.print("Running Viterbi..."); String testFilename = "punc.test"; - PuncConverter.convert("punc.ts", testFilename); + PuncConverter.convert(tsFilename, testFilename); DataSet testData = readTagged(testFilename); long startTime = System.currentTimeMillis(); highOrderSemiCrfModel.runViterbi(testData.getSeqList()); @@ -181,12 +181,12 @@ public void test() throws Exception { public static void main(String argv[]) throws Exception { PunctuationPredictor puncPredictor = new PunctuationPredictor(argv[1]); if (argv[0].toLowerCase().equals("all")) { - puncPredictor.train(); - puncPredictor.test(); + puncPredictor.train(argv[2]); + puncPredictor.test(argv[3]); } else if (argv[0].toLowerCase().equals("train")) { - puncPredictor.train(); + puncPredictor.train(argv[2]); } else if (argv[0].toLowerCase().equals("test")) { - puncPredictor.test(); + puncPredictor.test(argv[3]); } } } diff --git a/src/Applications/ReferenceTagger.java b/src/Applications/ReferenceTagger.java index acc5d43..3bd1ba6 100644 --- a/src/Applications/ReferenceTagger.java +++ b/src/Applications/ReferenceTagger.java @@ -51,7 +51,7 @@ public ReferenceTagger(String filename) { public DataSet readTagged(String filename) throws IOException { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; @@ -118,9 +118,9 @@ public void createFeatureGenerator() throws Exception { /** * Train the high-order semi-CRF. */ - public void train() throws Exception { + public void train(String trainFilename) throws Exception { // Set training file name and create output directory - String trainFilename = "ref.train"; +// String trainFilename = "ref.train"; File dir = new File("learntModels/"); dir.mkdirs(); @@ -142,7 +142,7 @@ public void train() throws Exception { /** * Test the high-order semi-CRF. */ - public void test() throws Exception { + public void test(String testFilename) throws Exception { // Read label map, features, and CRF model labelmap.read("learntModels/labelmap"); createFeatureGenerator(); @@ -152,7 +152,7 @@ public void test() throws Exception { // Run Viterbi algorithm System.out.print("Running Viterbi..."); - String testFilename = "ref.test"; +// String testFilename = "ref.test"; DataSet testData = readTagged(testFilename); long startTime = System.currentTimeMillis(); highOrderSemiCrfModel.runViterbi(testData.getSeqList()); @@ -178,12 +178,12 @@ public void test() throws Exception { public static void main(String argv[]) throws Exception { ReferenceTagger refTagger = new ReferenceTagger(argv[1]); if (argv[0].toLowerCase().equals("all")) { - refTagger.train(); - refTagger.test(); + refTagger.train(argv[2]); + refTagger.test(argv[3]); } else if (argv[0].toLowerCase().equals("train")) { - refTagger.train(); + refTagger.train(argv[2]); } else if (argv[0].toLowerCase().equals("test")) { - refTagger.test(); + refTagger.test(argv[3]); } } } diff --git a/src/HOCRF/FeatureGenerator.java b/src/HOCRF/FeatureGenerator.java index 3ddaa22..e0331de 100644 --- a/src/HOCRF/FeatureGenerator.java +++ b/src/HOCRF/FeatureGenerator.java @@ -21,6 +21,7 @@ import java.io.*; import java.util.*; + import Parallel.*; /** @@ -33,16 +34,16 @@ public class FeatureGenerator { int maxOrder; // Maximum order of the CRF Params params; // Parameters - HashMap obsMap; // Map from feature observation to its ID - HashMap patternMap; // Map from feature pattern to index - HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector + HashMap obsMap; // Map from feature observation to its ID + HashMap patternMap; // Map from feature pattern to index + HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector ArrayList featureList; // Map from feature ID to features - HashMap forwardStateMap; // Map from forward state to index + HashMap forwardStateMap; // Map from forward state to index ArrayList[] forwardTransition1; // Map from piID to list of pkID (see paper) ArrayList[] forwardTransition2; // Map from piID to list of pkyID (see paper) - HashMap backwardStateMap; // Map from backward state to index + HashMap backwardStateMap; // Map from backward state to index int[][] backwardTransition; // Map from [siID,y] to skID (see paper) ArrayList[] allSuffixes; // Map from sID to its suffixes patID ArrayList backwardStateList; // List of backward states @@ -66,7 +67,7 @@ public FeatureGenerator(ArrayList fts, Params pr) { * This method needs to be called before the training process. * @param trainData List of training sequences */ - public void initialize(ArrayList trainData) throws Exception { + public void initialize(ArrayList trainData) throws Exception { generateFeatureMap(trainData); generateForwardStatesMap(); generateBackwardStatesMap(); @@ -85,7 +86,7 @@ public void write(String filename) throws Exception { // Write observation map out.println(obsMap.size()); - Iterator iter = obsMap.keySet().iterator(); + Iterator iter = obsMap.keySet().iterator(); while (iter.hasNext()) { String key = (String) iter.next(); out.println(key + " " + obsMap.get(key)); @@ -101,9 +102,9 @@ public void write(String filename) throws Exception { // Write feature map out.println(featureMap.size()); - iter = featureMap.keySet().iterator(); - while (iter.hasNext()) { - FeatureIndex fi = (FeatureIndex) iter.next(); + Iterator iter_fi = featureMap.keySet().iterator(); + while (iter_fi.hasNext()) { + FeatureIndex fi = (FeatureIndex) iter_fi.next(); int index = (Integer) featureMap.get(fi); Feature f = featureList.get(index); out.println(f.obs + " " + f.pat + " " + f.value + " " + index); @@ -139,7 +140,7 @@ public void read(String filename) throws Exception { // Read observation map int mapSize = Integer.parseInt(in.readLine()); - obsMap = new HashMap(); + obsMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -150,7 +151,7 @@ public void read(String filename) throws Exception { // Read pattern map mapSize = Integer.parseInt(in.readLine()); - patternMap = new HashMap(); + patternMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -161,7 +162,7 @@ public void read(String filename) throws Exception { // Read feature map mapSize = Integer.parseInt(in.readLine()); - featureMap = new HashMap(); + featureMap = new HashMap<>(); featureList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) featureList.add(null); for (int i = 0; i < mapSize; i++) { @@ -178,7 +179,7 @@ public void read(String filename) throws Exception { // Read forward state map mapSize = Integer.parseInt(in.readLine()); - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < mapSize-1; i++) { String line = in.readLine(); @@ -190,7 +191,7 @@ public void read(String filename) throws Exception { // Read backward state map mapSize = Integer.parseInt(in.readLine()); - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) backwardStateList.add(null); for (int i = 0; i < mapSize; i++) { @@ -278,7 +279,7 @@ public int getMaxOrder() { * Generate the features for each training sequence. * @param trainData List of training sequences */ - public void generateSentenceFeat(ArrayList trainData) throws Exception { + public void generateSentenceFeat(ArrayList trainData) throws Exception { SentenceFeatGenerator gen = new SentenceFeatGenerator(trainData, this); Scheduler sch = new Scheduler(gen, params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); @@ -288,10 +289,10 @@ public void generateSentenceFeat(ArrayList trainData) throws Exception { * Generate the observation map, pattern map, feature map, and feature list from training data. * @param trainData List of training sequences */ - public void generateFeatureMap(ArrayList trainData) { - obsMap = new HashMap(); - patternMap = new HashMap(); - featureMap = new HashMap(); + public void generateFeatureMap(ArrayList trainData) { + obsMap = new HashMap<>(); + patternMap = new HashMap<>(); + featureMap = new HashMap<>(); featureList = new ArrayList(); for (int t = 0; t < trainData.size(); t++) { DataSequence seq = (DataSequence) trainData.get(t); @@ -326,12 +327,12 @@ public void generateFeatureMap(ArrayList trainData) { * Generate the forward state map. */ public void generateForwardStatesMap() { - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < params.numLabels; i++) { forwardStateMap.put("" + i, new Integer(forwardStateMap.size())); } - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { String labelPat = (String) iter.next(); ArrayList pats = Utility.generateProperPrefixes(labelPat); @@ -347,12 +348,12 @@ public void generateForwardStatesMap() { * Generate the backward state map and the backward state list. */ public void generateBackwardStatesMap() { - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(); - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String p = (String) iter.next(); - int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); +// int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); for (int y = 0; y < params.numLabels; y++) { String py = p.equals("") ? y + "" : y + "|" + p; if (getBackwardStateIndex(py) == null) { @@ -364,7 +365,7 @@ public void generateBackwardStatesMap() { } /** - * Generate the maximum posible pattern for a position. + * Generate the maximum possible pattern for a position. * Note that patterns are in reversed order: y(t)|y(t-1)|y(t-2)|... * @param seq Data sequence * @param pos Input position @@ -432,7 +433,7 @@ public ArrayList generateObs(DataSequence seq, int pos) { * @param map Map from strings to indices * @return Index of the longest suffix of the input string from the input map. */ - public Integer getLongestSuffixID(String p, HashMap map) { + public Integer getLongestSuffixID(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -449,7 +450,7 @@ public Integer getLongestSuffixID(String p, HashMap map) { * @param map Map from strings to indices * @return The longest suffix of the input string from the input map. */ - public String getLongestSuffix(String p, HashMap map) { + public String getLongestSuffix(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -463,11 +464,12 @@ public String getLongestSuffix(String p, HashMap map) { /** * Build the information for the forward algorithm. */ - public void buildForwardTransition() { + @SuppressWarnings("unchecked") + public void buildForwardTransition() { forwardTransition1 = new ArrayList[forwardStateMap.size()]; forwardTransition2 = new ArrayList[forwardStateMap.size()]; - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String pk = (String) iter.next(); int pkID = getForwardStateIndex(pk); @@ -488,15 +490,16 @@ public void buildForwardTransition() { /** * Build the information for the backward algorithm. */ - public void buildBackwardTransition() { + @SuppressWarnings("unchecked") + public void buildBackwardTransition() { backwardTransition = new int[backwardStateMap.size()][params.numLabels]; allSuffixes = new ArrayList[backwardStateMap.size()]; - Iterator iter = backwardStateMap.keySet().iterator(); + Iterator iter = backwardStateMap.keySet().iterator(); while (iter.hasNext()) { String si = (String) iter.next(); int siID = getBackwardStateIndex(si); - int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); +// int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); for (int y = 0; y < params.numLabels; y++) { String siy = y + "|" + si; String sk = getLongestSuffix(siy, backwardStateMap); @@ -517,14 +520,15 @@ public void buildBackwardTransition() { /** * Build the information to compute the marginals and expected feature scores. */ - public void buildPatternTransition() { + @SuppressWarnings("unchecked") + public void buildPatternTransition() { patternTransition1 = new ArrayList[patternMap.size()]; patternTransition2 = new ArrayList[patternMap.size()]; - Iterator forwardIter = forwardStateMap.keySet().iterator(); + Iterator forwardIter = forwardStateMap.keySet().iterator(); while (forwardIter.hasNext()) { String pi = (String) forwardIter.next(); - int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); +// int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); int piID = getForwardStateIndex(pi); for (int y = 0; y < params.numLabels; y++) { String piy = pi.equals("") ? y + "" : y + "|" + pi; diff --git a/src/HOCRF/Function.java b/src/HOCRF/Function.java index 291aafa..5661852 100644 --- a/src/HOCRF/Function.java +++ b/src/HOCRF/Function.java @@ -30,7 +30,7 @@ public class Function implements DiffFunction { FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences // Private data structures to compute function value and derivatives private Loglikelihood logli; // Loglikelihood values @@ -41,7 +41,7 @@ public class Function implements DiffFunction { * @param fgen Feature generator * @param data Training data */ - public Function(FeatureGenerator fgen, ArrayList data) { + public Function(FeatureGenerator fgen, ArrayList data) { featureGen = fgen; trainData = data; lambdaCache = null; diff --git a/src/HOCRF/HighOrderCRF.java b/src/HOCRF/HighOrderCRF.java index 072fe7b..7c7ac4f 100644 --- a/src/HOCRF/HighOrderCRF.java +++ b/src/HOCRF/HighOrderCRF.java @@ -47,7 +47,7 @@ public HighOrderCRF(FeatureGenerator fgen) { * Train a high-order CRF from data. * @param data Training data */ - public void train(ArrayList data) { + public void train(ArrayList data) { QNMinimizer qn = new QNMinimizer(); Function df = new Function(featureGen, data); lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); @@ -57,7 +57,7 @@ public void train(ArrayList data) { * Run Viterbi algorithm on testing data. * @param data Testing data */ - public void runViterbi(ArrayList data) throws Exception { + public void runViterbi(ArrayList data) throws Exception { Viterbi tester = new Viterbi(featureGen, lambda, data); Scheduler sch = new Scheduler(tester, featureGen.params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); diff --git a/src/HOCRF/LogliComputer.java b/src/HOCRF/LogliComputer.java index eefa2e3..f14e85f 100644 --- a/src/HOCRF/LogliComputer.java +++ b/src/HOCRF/LogliComputer.java @@ -30,7 +30,7 @@ public class LogliComputer implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences double[] lambda; // Lambda vector Loglikelihood logli; // Loglikelihood value and derivatives final int BASE = 1; // Base of the logAlpha array @@ -42,7 +42,7 @@ public class LogliComputer implements Schedulable { * @param td List of training sequences * @param loglh Initial loglikelihood and its derivatives (partially computed from class Function) */ - public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { + public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { curID = -1; featureGen = fgen; trainData = td; @@ -57,7 +57,7 @@ public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, */ public Object compute(int taskID) { Loglikelihood res = new Loglikelihood(lambda.length); - DataSequence seq = (DataSequence) trainData.get(taskID); + DataSequence seq = trainData.get(taskID); addFeatureScores(seq, res); double[][] logAlpha = computeLogAlpha(seq); diff --git a/src/HOCRF/Scorer.java b/src/HOCRF/Scorer.java index 3b6c47f..4ba31ba 100644 --- a/src/HOCRF/Scorer.java +++ b/src/HOCRF/Scorer.java @@ -19,9 +19,10 @@ package HOCRF; -import java.io.*; -import java.util.*; -import java.text.*; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; /** * Scorer class @@ -40,16 +41,16 @@ public class Scorer { * @param labelmap Label map * @param RM_SUFFIX If set to true, suffixes of labels after '-' will be removed */ - public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { + public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { labels = new String[trueData.size()][]; for (int i = 0; i < trueData.size(); i++) { - DataSequence seq = (DataSequence) trueData.get(i); + DataSequence seq = trueData.get(i); labels[i] = labelmap.revArray(seq.labels); } predicted = new String[predictedData.size()][]; for (int i = 0; i < predictedData.size(); i++) { - DataSequence seq = (DataSequence) predictedData.get(i); + DataSequence seq = predictedData.get(i); predicted[i] = labelmap.revArray(seq.labels); } diff --git a/src/HOCRF/SentenceFeatGenerator.java b/src/HOCRF/SentenceFeatGenerator.java index 482d89b..eca4119 100644 --- a/src/HOCRF/SentenceFeatGenerator.java +++ b/src/HOCRF/SentenceFeatGenerator.java @@ -20,6 +20,7 @@ package HOCRF; import java.util.*; + import Parallel.*; /** @@ -30,7 +31,7 @@ public class SentenceFeatGenerator implements Schedulable { int curID; // Current task ID (for parallelization) - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences FeatureGenerator featGen; // Feature generator /** @@ -38,7 +39,7 @@ public class SentenceFeatGenerator implements Schedulable { * @param data Training data * @param fgen Feature generator */ - public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { + public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { curID = -1; trainData = data; featGen = fgen; @@ -49,8 +50,9 @@ public SentenceFeatGenerator(ArrayList data, FeatureGenerator fgen) { * @param taskID Index of the training sequence * @return The updated sequence */ - public Object compute(int taskID) { - DataSequence seq = (DataSequence) trainData.get(taskID); + @SuppressWarnings("unchecked") + public Object compute(int taskID) { + DataSequence seq = trainData.get(taskID); seq.features = new ArrayList[seq.length()][featGen.patternMap.size()]; for (int pos = 0; pos < seq.length(); pos++) { diff --git a/src/HOCRF/Viterbi.java b/src/HOCRF/Viterbi.java index 71526b6..d622d40 100644 --- a/src/HOCRF/Viterbi.java +++ b/src/HOCRF/Viterbi.java @@ -32,7 +32,7 @@ public class Viterbi implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator double[] lambda; // Lambda vector - ArrayList data; // List of testing sequences + ArrayList data; // List of testing sequences final int BASE = 1; // Base of the logAlpha array /** @@ -41,7 +41,7 @@ public class Viterbi implements Schedulable { * @param lambda Lambda vector * @param data Testing data */ - public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { + public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { curID = -1; this.featureGen = featureGen; this.lambda = lambda; @@ -54,7 +54,7 @@ public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { * @return The updated sequence */ public Object compute(int taskID) { - DataSequence seq = (DataSequence) data.get(taskID); + DataSequence seq = data.get(taskID); double maxScore[][] = new double[seq.length() + 1][featureGen.forwardStateMap.size()]; String trace[][] = new String[seq.length()][featureGen.forwardStateMap.size()]; diff --git a/src/HOSemiCRF/DataSequence.java b/src/HOSemiCRF/DataSequence.java index 0ba49bb..569274c 100644 --- a/src/HOSemiCRF/DataSequence.java +++ b/src/HOSemiCRF/DataSequence.java @@ -19,8 +19,7 @@ package HOSemiCRF; -import java.io.*; -import java.util.*; +import java.io.BufferedWriter; /** * Class for a data sequence diff --git a/src/HOSemiCRF/FeatureGenerator.java b/src/HOSemiCRF/FeatureGenerator.java index 0e52000..2e4f54c 100644 --- a/src/HOSemiCRF/FeatureGenerator.java +++ b/src/HOSemiCRF/FeatureGenerator.java @@ -21,6 +21,7 @@ import java.io.*; import java.util.*; + import Parallel.*; /** @@ -35,17 +36,17 @@ public class FeatureGenerator { Params params; // Parameters int[] maxMemory; // Maximum segment length for each label - HashMap obsMap; // Map from feature observation to its ID - HashMap patternMap; // Map from feature pattern to index - HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector + HashMap obsMap; // Map from feature observation to its ID + HashMap patternMap; // Map from feature pattern to index + HashMap featureMap; // Map from FeatureIndex to its ID in lambda vector ArrayList featureList; // Map from feature ID to features - HashMap forwardStateMap; // Map from forward state to index + HashMap forwardStateMap; // Map from forward state to index ArrayList[] forwardTransition1; // Map from piID to list of pkID (see paper) ArrayList[] forwardTransition2; // Map from piID to list of pkyID (see paper) int[] lastForwardStateLabel; // Map from piID to its last label - HashMap backwardStateMap; // Map from backward state to index + HashMap backwardStateMap; // Map from backward state to index int[][] backwardTransition; // Map from [siID,y] to skID (see paper) ArrayList[] allSuffixes; // Map from sID to its suffixes patID ArrayList backwardStateList; @@ -71,7 +72,7 @@ public FeatureGenerator(ArrayList fts, Params pr) { * This method needs to be called before the training process. * @param trainData List of training sequences */ - public void initialize(ArrayList trainData) throws Exception { + public void initialize(ArrayList trainData) throws Exception { createMaxMemory(trainData); generateFeatureMap(trainData); generateForwardStatesMap(); @@ -91,7 +92,7 @@ public void write(String filename) throws Exception { // Write observation map out.println(obsMap.size()); - Iterator iter = obsMap.keySet().iterator(); + Iterator iter = obsMap.keySet().iterator(); while (iter.hasNext()) { String key = (String) iter.next(); out.println(key + " " + obsMap.get(key)); @@ -107,9 +108,9 @@ public void write(String filename) throws Exception { // Write feature map out.println(featureMap.size()); - iter = featureMap.keySet().iterator(); - while (iter.hasNext()) { - FeatureIndex fi = (FeatureIndex) iter.next(); + Iterator fIter = featureMap.keySet().iterator(); + while (fIter.hasNext()) { + FeatureIndex fi = (FeatureIndex) fIter.next(); int index = (Integer) featureMap.get(fi); Feature f = featureList.get(index); out.println(f.obs + " " + f.pat + " " + f.value + " " + index); @@ -150,7 +151,7 @@ public void read(String filename) throws Exception { // Read observation map int mapSize = Integer.parseInt(in.readLine()); - obsMap = new HashMap(); + obsMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -161,7 +162,7 @@ public void read(String filename) throws Exception { // Read pattern map mapSize = Integer.parseInt(in.readLine()); - patternMap = new HashMap(); + patternMap = new HashMap<>(); for (int i = 0; i < mapSize; i++) { String line = in.readLine(); StringTokenizer toks = new StringTokenizer(line); @@ -172,7 +173,7 @@ public void read(String filename) throws Exception { // Read feature map mapSize = Integer.parseInt(in.readLine()); - featureMap = new HashMap(); + featureMap = new HashMap<>(); featureList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) featureList.add(null); for (int i = 0; i < mapSize; i++) { @@ -189,7 +190,7 @@ public void read(String filename) throws Exception { // Read forward state map mapSize = Integer.parseInt(in.readLine()); - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < mapSize-1; i++) { String line = in.readLine(); @@ -201,7 +202,7 @@ public void read(String filename) throws Exception { // Read backward state map mapSize = Integer.parseInt(in.readLine()); - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(mapSize); for (int i = 0; i < mapSize; i++) backwardStateList.add(null); for (int i = 0; i < mapSize; i++) { @@ -295,7 +296,7 @@ public int getMaxOrder() { * Generate the observations for each training sequence. * @param trainData List of training sequences */ - public void generateSentenceObs(ArrayList trainData) throws Exception { + public void generateSentenceObs(ArrayList trainData) throws Exception { SentenceObsGenerator gen = new SentenceObsGenerator(trainData, this); Scheduler sch = new Scheduler(gen, params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); @@ -307,7 +308,7 @@ public void generateSentenceObs(ArrayList trainData) throws Exception { * Reset the segment information for each training sequence. * @param trainData List of training sequences */ - public void createMaxMemory(ArrayList trainData) throws Exception { + public void createMaxMemory(ArrayList trainData) throws Exception { maxMemory = new int[params.numLabels]; Arrays.fill(maxMemory, -1); @@ -342,10 +343,10 @@ public void createMaxMemory(ArrayList trainData) throws Exception { * Generate the observation map, pattern map, feature map, and feature list from training data. * @param trainData List of training sequences */ - public void generateFeatureMap(ArrayList trainData) { - obsMap = new HashMap(); - patternMap = new HashMap(); - featureMap = new HashMap(); + public void generateFeatureMap(ArrayList trainData) { + obsMap = new HashMap<>(); + patternMap = new HashMap<>(); + featureMap = new HashMap<>(); featureList = new ArrayList(); for (int t = 0; t < trainData.size(); t++) { DataSequence seq = (DataSequence) trainData.get(t); @@ -381,12 +382,12 @@ public void generateFeatureMap(ArrayList trainData) { * Generate the forward state map. */ public void generateForwardStatesMap() { - forwardStateMap = new HashMap(); + forwardStateMap = new HashMap<>(); forwardStateMap.put("", new Integer(0)); for (int i = 0; i < params.numLabels; i++) { forwardStateMap.put("" + i, new Integer(forwardStateMap.size())); } - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { String labelPat = (String) iter.next(); ArrayList pats = Utility.generateProperPrefixes(labelPat); @@ -402,9 +403,9 @@ public void generateForwardStatesMap() { * Generate the backward state map and the backward state list. */ public void generateBackwardStatesMap() { - backwardStateMap = new HashMap(); + backwardStateMap = new HashMap<>(); backwardStateList = new ArrayList(); - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { String p = (String) iter.next(); int lastLabel = p.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(p)); @@ -499,7 +500,7 @@ public ArrayList generateObs(DataSequence seq, int segStart, int segEnd) * @param map Map from strings to indices * @return Index of the longest suffix of the input string from the input map. */ - public Integer getLongestSuffixID(String p, HashMap map) { + public Integer getLongestSuffixID(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -516,7 +517,7 @@ public Integer getLongestSuffixID(String p, HashMap map) { * @param map Map from strings to indices * @return The longest suffix of the input string from the input map. */ - public String getLongestSuffix(String p, HashMap map) { + public String getLongestSuffix(String p, HashMap map) { ArrayList suffixes = Utility.generateSuffixes(p); for (int i = 0; i < suffixes.size(); i++) { Integer index = (Integer) map.get(suffixes.get(i)); @@ -530,14 +531,15 @@ public String getLongestSuffix(String p, HashMap map) { /** * Build the information for the forward algorithm. */ - public void buildForwardTransition() { + @SuppressWarnings("unchecked") + public void buildForwardTransition() { forwardTransition1 = new ArrayList[forwardStateMap.size()]; forwardTransition2 = new ArrayList[forwardStateMap.size()]; lastForwardStateLabel = new int[forwardStateMap.size()]; - Iterator iter = forwardStateMap.keySet().iterator(); + Iterator iter = forwardStateMap.keySet().iterator(); while (iter.hasNext()) { - String pk = (String) iter.next(); + String pk = iter.next(); int pkID = getForwardStateIndex(pk); lastForwardStateLabel[pkID] = pk.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pk)); @@ -559,13 +561,14 @@ public void buildForwardTransition() { /** * Build the information for the backward algorithm. */ - public void buildBackwardTransition() { + @SuppressWarnings("unchecked") + public void buildBackwardTransition() { backwardTransition = new int[backwardStateMap.size()][params.numLabels]; allSuffixes = new ArrayList[backwardStateMap.size()]; - Iterator iter = backwardStateMap.keySet().iterator(); + Iterator iter = backwardStateMap.keySet().iterator(); while (iter.hasNext()) { - String si = (String) iter.next(); + String si = iter.next(); int siID = getBackwardStateIndex(si); int lastLabel = si.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(si)); for (int y = 0; y < params.numLabels; y++) { @@ -592,15 +595,16 @@ public void buildBackwardTransition() { /** * Build the information to compute the marginals and expected feature scores. */ - public void buildPatternTransition() { + @SuppressWarnings("unchecked") + public void buildPatternTransition() { patternTransition1 = new ArrayList[patternMap.size()]; patternTransition2 = new ArrayList[patternMap.size()]; lastPatternLabel = new int[patternMap.size()]; patternBackwardID = new int[patternMap.size()]; - Iterator iter = patternMap.keySet().iterator(); + Iterator iter = patternMap.keySet().iterator(); while (iter.hasNext()) { - String p = (String) iter.next(); + String p = iter.next(); int pID = getPatternIndex(p).intValue(); patternBackwardID[pID] = getBackwardStateIndex(p); String lastY = Utility.getLastLabel(p); @@ -611,9 +615,9 @@ public void buildPatternTransition() { } } - Iterator forwardIter = forwardStateMap.keySet().iterator(); + Iterator forwardIter = forwardStateMap.keySet().iterator(); while (forwardIter.hasNext()) { - String pi = (String) forwardIter.next(); + String pi = forwardIter.next(); int lastLabel = pi.equals("") ? -1 : Integer.parseInt(Utility.getLastLabel(pi)); int piID = getForwardStateIndex(pi); for (int y = 0; y < params.numLabels; y++) { diff --git a/src/HOSemiCRF/Function.java b/src/HOSemiCRF/Function.java index 0a52bb7..19b1d1d 100644 --- a/src/HOSemiCRF/Function.java +++ b/src/HOSemiCRF/Function.java @@ -30,7 +30,7 @@ public class Function implements DiffFunction { FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences // Private data structures to compute function value and derivatives private Loglikelihood logli; // Loglikelihood values @@ -41,7 +41,7 @@ public class Function implements DiffFunction { * @param fgen Feature generator * @param data Training data */ - public Function(FeatureGenerator fgen, ArrayList data) { + public Function(FeatureGenerator fgen, ArrayList data) { featureGen = fgen; trainData = data; lambdaCache = null; diff --git a/src/HOSemiCRF/HighOrderSemiCRF.java b/src/HOSemiCRF/HighOrderSemiCRF.java index 2294770..c533127 100644 --- a/src/HOSemiCRF/HighOrderSemiCRF.java +++ b/src/HOSemiCRF/HighOrderSemiCRF.java @@ -47,7 +47,7 @@ public HighOrderSemiCRF(FeatureGenerator fgen) { * Train a high-order semi-CRF from data. * @param data Training data */ - public void train(ArrayList data) { + public void train(ArrayList data) { QNMinimizer qn = new QNMinimizer(); Function df = new Function(featureGen, data); lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); @@ -57,7 +57,7 @@ public void train(ArrayList data) { * Run Viterbi algorithm on testing data. * @param data Testing data */ - public void runViterbi(ArrayList data) throws Exception { + public void runViterbi(ArrayList data) throws Exception { Viterbi tester = new Viterbi(featureGen, lambda, data); Scheduler sch = new Scheduler(tester, featureGen.params.numthreads, Scheduler.DYNAMIC_NEXT_AVAILABLE); sch.run(); diff --git a/src/HOSemiCRF/LogliComputer.java b/src/HOSemiCRF/LogliComputer.java index 53946d7..fb71418 100644 --- a/src/HOSemiCRF/LogliComputer.java +++ b/src/HOSemiCRF/LogliComputer.java @@ -30,7 +30,7 @@ public class LogliComputer implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences double[] lambda; // Lambda vector Loglikelihood logli; // Loglikelihood value and derivatives final int BASE = 1; // Base of the logAlpha array @@ -42,7 +42,7 @@ public class LogliComputer implements Schedulable { * @param td List of training sequences * @param loglh Initial loglikelihood and its derivatives (partially computed from class Function) */ - public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { + public LogliComputer(double[] lambdaValues, FeatureGenerator fgen, ArrayList td, Loglikelihood loglh) { curID = -1; featureGen = fgen; trainData = td; diff --git a/src/HOSemiCRF/Scorer.java b/src/HOSemiCRF/Scorer.java index 208ff33..c4f7fc1 100644 --- a/src/HOSemiCRF/Scorer.java +++ b/src/HOSemiCRF/Scorer.java @@ -19,9 +19,11 @@ package HOSemiCRF; -import java.io.*; -import java.util.*; -import java.text.*; +import java.text.DecimalFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Hashtable; +import java.util.Vector; /** * Scorer class @@ -40,7 +42,7 @@ public class Scorer { * @param labelmap Label map * @param RM_SUFFIX If set to true, suffixes of labels after '-' will be removed */ - public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { + public Scorer(ArrayList trueData, ArrayList predictedData, LabelMap labelmap, boolean RM_SUFFIX) { labels = new String[trueData.size()][]; for (int i = 0; i < trueData.size(); i++) { DataSequence seq = (DataSequence) trueData.get(i); diff --git a/src/HOSemiCRF/SentenceObsGenerator.java b/src/HOSemiCRF/SentenceObsGenerator.java index d1128df..fb7c994 100644 --- a/src/HOSemiCRF/SentenceObsGenerator.java +++ b/src/HOSemiCRF/SentenceObsGenerator.java @@ -30,7 +30,7 @@ public class SentenceObsGenerator implements Schedulable { int curID; // Current task ID (for parallelization) - ArrayList trainData; // List of training sequences + ArrayList trainData; // List of training sequences FeatureGenerator featGen; // Feature generator /** @@ -38,7 +38,7 @@ public class SentenceObsGenerator implements Schedulable { * @param data Training data * @param fgen Feature generator */ - public SentenceObsGenerator(ArrayList data, FeatureGenerator fgen) { + public SentenceObsGenerator(ArrayList data, FeatureGenerator fgen) { curID = -1; trainData = data; featGen = fgen; diff --git a/src/HOSemiCRF/Viterbi.java b/src/HOSemiCRF/Viterbi.java index 5ecd1fa..2b47958 100644 --- a/src/HOSemiCRF/Viterbi.java +++ b/src/HOSemiCRF/Viterbi.java @@ -32,7 +32,7 @@ public class Viterbi implements Schedulable { int curID; // Current task ID (for parallelization) FeatureGenerator featureGen; // Feature generator double[] lambda; // Lambda vector - ArrayList data; // List of training sequences + ArrayList data; // List of training sequences final int BASE = 1; // Base of the logAlpha array /** @@ -41,7 +41,7 @@ public class Viterbi implements Schedulable { * @param lambda Lambda vector * @param data Training data */ - public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { + public Viterbi(FeatureGenerator featureGen, double[] lambda, ArrayList data) { curID = -1; this.featureGen = featureGen; this.lambda = lambda; diff --git a/src/OCR/OCR.java b/src/OCR/OCR.java index 8d1ae2f..081de57 100644 --- a/src/OCR/OCR.java +++ b/src/OCR/OCR.java @@ -44,7 +44,7 @@ public OCR(String filename, String fold) { public DataSet readTagged(String filename, int trainFold, boolean isTraining) throws Exception { BufferedReader in = new BufferedReader(new FileReader(filename)); - ArrayList td = new ArrayList(); + ArrayList td = new ArrayList<>(); ArrayList inps = new ArrayList(); ArrayList labels = new ArrayList(); String line; diff --git a/src/Parallel/Scheduler.java b/src/Parallel/Scheduler.java index a8830ee..0f17bf8 100644 --- a/src/Parallel/Scheduler.java +++ b/src/Parallel/Scheduler.java @@ -99,7 +99,7 @@ class SimpleTask implements Schedulable { double[] ans; int curID; - static final int N = 1000000; +// static final int N = 1000000; int nTasks; public SimpleTask(int nTasks) { @@ -108,7 +108,8 @@ public SimpleTask(int nTasks) { ans = new double[2]; } - public void showResult() { + @SuppressWarnings("unused") + public void showResult() { System.out.println(ans[0] + " " + ans[1]); } @@ -129,7 +130,8 @@ public Object compute(int taskID) { return result; } - public int getNumCompletedTasks() { + @SuppressWarnings("unused") + public int getNumCompletedTasks() { return curID; } From d7e986ace6923b4d1c84e6a63738fab67c157c41 Mon Sep 17 00:00:00 2001 From: tndoan Date: Sat, 25 Jul 2015 23:16:00 +0800 Subject: [PATCH 02/12] add getter --- src/HOSemiCRF/Params.java | 48 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/HOSemiCRF/Params.java b/src/HOSemiCRF/Params.java index ac68320..24cc3ea 100644 --- a/src/HOSemiCRF/Params.java +++ b/src/HOSemiCRF/Params.java @@ -61,4 +61,52 @@ public Params(String filename, int nl) throws IOException { } numLabels = nl; } + + /** + * + * @return + */ + public int getNumLabels() { + return numLabels; + } + + /** + * + * @return + */ + public int getMaxIters() { + return maxIters; + } + + /** + * + * @return + */ + public int getNumthreads() { + return numthreads; + } + + /** + * + * @return + */ + public int getMaxSegment() { + return maxSegment; + } + + /** + * + * @return + */ + public double getInvSigmaSquare() { + return invSigmaSquare; + } + + /** + * + * @return + */ + public double getEpsForConvergence() { + return epsForConvergence; + } } From 23c59202120f945a75d746484bdfdedd87ea43f2 Mon Sep 17 00:00:00 2001 From: tndoan Date: Sat, 25 Jul 2015 23:16:22 +0800 Subject: [PATCH 03/12] add getter of params --- src/HOSemiCRF/FeatureGenerator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/HOSemiCRF/FeatureGenerator.java b/src/HOSemiCRF/FeatureGenerator.java index 2e4f54c..c307238 100644 --- a/src/HOSemiCRF/FeatureGenerator.java +++ b/src/HOSemiCRF/FeatureGenerator.java @@ -723,4 +723,12 @@ public void printStatesStatistics() { } } } + + /** + * + * @return object which contains all value of parameters. + */ + public Params getParams() { + return params; + } } From b54a8302042884852cbbc493623a0733e8be17ab Mon Sep 17 00:00:00 2001 From: tndoan Date: Sun, 26 Jul 2015 16:30:21 +0800 Subject: [PATCH 04/12] add getters and constructor --- src/HOSemiCRF/Loglikelihood.java | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/HOSemiCRF/Loglikelihood.java b/src/HOSemiCRF/Loglikelihood.java index febf0c3..d7fa82a 100644 --- a/src/HOSemiCRF/Loglikelihood.java +++ b/src/HOSemiCRF/Loglikelihood.java @@ -22,6 +22,7 @@ /** * Loglikelihood class * @author Nguyen Viet Cuong + * @author tndoan */ public class Loglikelihood { @@ -36,4 +37,31 @@ public Loglikelihood(int n) { logli = 0; derivatives = new double[n]; } + + /** + * construct loglikelihood with initial loglikelihood and derivative + * @param logli initial loglikelihood + * @param dev initial derivative + */ + public Loglikelihood(double logli, double[] dev) { + this.logli = logli; + derivatives = new double[dev.length]; + System.arraycopy(dev, 0, derivatives, 0, dev.length); + } + + /** + * + * @return the log likelihood + */ + public double getLogli() { + return logli; + } + + /** + * + * @return the derivative + */ + public double[] getDerivatives() { + return derivatives; + } } From 8cfe30e42fcb6f7f77e4525daeb9901fdb272d33 Mon Sep 17 00:00:00 2001 From: tndoan Date: Mon, 27 Jul 2015 20:44:28 +0800 Subject: [PATCH 05/12] add getter to return the size of features --- src/HOSemiCRF/FeatureGenerator.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/HOSemiCRF/FeatureGenerator.java b/src/HOSemiCRF/FeatureGenerator.java index c307238..595f481 100644 --- a/src/HOSemiCRF/FeatureGenerator.java +++ b/src/HOSemiCRF/FeatureGenerator.java @@ -731,4 +731,12 @@ public void printStatesStatistics() { public Params getParams() { return params; } + + /** + * + * @return the dimension of features + */ + public int getFeatureSize(){ + return featureMap.size(); + } } From b6971b449af1ef70445acdb15bd691c462d227d1 Mon Sep 17 00:00:00 2001 From: tndoan Date: Tue, 28 Jul 2015 14:28:51 +0800 Subject: [PATCH 06/12] add learning rate as parameters of file --- src/HOSemiCRF/Params.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/HOSemiCRF/Params.java b/src/HOSemiCRF/Params.java index 24cc3ea..1bf5632 100644 --- a/src/HOSemiCRF/Params.java +++ b/src/HOSemiCRF/Params.java @@ -25,6 +25,7 @@ /** * Parameters class * @author Nguyen Viet Cuong + * @author tndoan */ public class Params { @@ -34,6 +35,11 @@ public class Params { int maxSegment = -1; // Maximum segment length double invSigmaSquare = 1.0; // Inverse of Sigma Squared double epsForConvergence = 0.001; // Convergence Precision + + /** + * learning rate for first order minimizer + */ + double learningRate = 0.1; // /** * Construct a parameters object. @@ -59,6 +65,9 @@ public Params(String filename, int nl) throws IOException { if ((value = options.getProperty("epsForConvergence")) != null) { epsForConvergence = Double.parseDouble(value); } + if ((value = options.getProperty("learningRate")) != null){ + learningRate = Double.parseDouble(value); + } numLabels = nl; } @@ -109,4 +118,12 @@ public double getInvSigmaSquare() { public double getEpsForConvergence() { return epsForConvergence; } + + /** + * + * @return the value of learning rate + */ + public double getLearningRate() { + return learningRate; + } } From 350c0b52cb943122075432b8d531b94baa92771d Mon Sep 17 00:00:00 2001 From: tndoan Date: Tue, 28 Jul 2015 14:29:39 +0800 Subject: [PATCH 07/12] change from using Quasi-Newton technique from Stanford NLP package to SVRG --- src/HOSemiCRF/HighOrderSemiCRF.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/HOSemiCRF/HighOrderSemiCRF.java b/src/HOSemiCRF/HighOrderSemiCRF.java index c533127..9c04576 100644 --- a/src/HOSemiCRF/HighOrderSemiCRF.java +++ b/src/HOSemiCRF/HighOrderSemiCRF.java @@ -21,7 +21,9 @@ import java.io.*; import java.util.*; -import edu.stanford.nlp.optimization.*; + +import optimization.FirstOrderDiffFunction; +import optimization.SVRGMinimizer; import Parallel.*; /** @@ -48,9 +50,14 @@ public HighOrderSemiCRF(FeatureGenerator fgen) { * @param data Training data */ public void train(ArrayList data) { - QNMinimizer qn = new QNMinimizer(); - Function df = new Function(featureGen, data); - lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); + // use library to do minimization +// QNMinimizer qn = new QNMinimizer(); +// Function df = new Function(featureGen, data); +// lambda = qn.minimize(df, featureGen.params.epsForConvergence, lambda, featureGen.params.maxIters); + + FirstOrderDiffFunction func = new FirstOrderDiffFunction(featureGen, data); + SVRGMinimizer svrg = new SVRGMinimizer(); + lambda = svrg.minimize(func, lambda, featureGen.params.getLearningRate(), featureGen.params.maxIters, featureGen.params.epsForConvergence); } /** From 0aaf49372ff4f985970d2511039d2de5d4d98262 Mon Sep 17 00:00:00 2001 From: tndoan Date: Tue, 28 Jul 2015 14:33:53 +0800 Subject: [PATCH 08/12] add implementation of functions --- src/optimization/AbstractSVRGFunction.java | 54 +++++++++ src/optimization/FirstOrderDiffFunction.java | 114 +++++++++++++++++++ src/optimization/SquareLossFunction.java | 101 ++++++++++++++++ 3 files changed, 269 insertions(+) create mode 100644 src/optimization/AbstractSVRGFunction.java create mode 100644 src/optimization/FirstOrderDiffFunction.java create mode 100644 src/optimization/SquareLossFunction.java diff --git a/src/optimization/AbstractSVRGFunction.java b/src/optimization/AbstractSVRGFunction.java new file mode 100644 index 0000000..48a97b6 --- /dev/null +++ b/src/optimization/AbstractSVRGFunction.java @@ -0,0 +1,54 @@ +package optimization; + +import java.util.ArrayList; + +import HOSemiCRF.DataSequence; + +/** + * It is the abstract of function + * @author tndoan + * + */ +public abstract class AbstractSVRGFunction { + // similar AbstractStochasticCachingDiffUpdateFunction + + protected ArrayList data; + + /** + * + * @return the number of data points + */ + public int getNumberOfDataPoints(){ + return data.size(); + } + + /** + * Get value of function + * @param w value of parameters + * @return value of function + */ + public abstract double valueAt(double[] w); + + /** + * take derivative of function of whole data + * @param w value of parameters + * @return array corresponding to the derivative + */ + public abstract double[] takeDerivative(double[] w); + + /** + * take derivative of function of data point whose indices are given + * @param w value of parameters + * @param index array which contains index of data point that we will use + * @return array of derivative + */ + public abstract double[] takeDerivative(double[] w, int[] index); + + /** + * take derivative of function of data point whose index is given + * @param w value of parameter + * @param index index of data point in array + * @return derivative of function with parameter w at data point whose index is specified + */ + public abstract double[] takeDerivative(double[] w, int index); +} \ No newline at end of file diff --git a/src/optimization/FirstOrderDiffFunction.java b/src/optimization/FirstOrderDiffFunction.java new file mode 100644 index 0000000..335cc3f --- /dev/null +++ b/src/optimization/FirstOrderDiffFunction.java @@ -0,0 +1,114 @@ +package optimization; + +import java.util.ArrayList; + +import HOSemiCRF.DataSequence; +import HOSemiCRF.FeatureGenerator; +import HOSemiCRF.LogliComputer; +import HOSemiCRF.Loglikelihood; +import Parallel.Scheduler; + +public class FirstOrderDiffFunction extends AbstractSVRGFunction { + + /** + * Feature Generator + */ + FeatureGenerator featureGen; // Feature generator + + public FirstOrderDiffFunction(FeatureGenerator fg, ArrayList data) { + this.data = data; + this.featureGen = fg; + } + + @Override + public double valueAt(double[] w) { + double logli = 0; + for (int i = 0; i < w.length; i++) { + logli -= ((w[i] * w[i]) * featureGen.getParams().getInvSigmaSquare()) / 2; + } + + Loglikelihood l = new Loglikelihood(logli, new double[w.length]); // we dont care about derivative + + LogliComputer logliComp = new LogliComputer(w, featureGen, data, l); + Scheduler sch = new Scheduler(logliComp, featureGen.getParams().getNumthreads(), Scheduler.DYNAMIC_NEXT_AVAILABLE); + try { + sch.run(); + } catch (Exception e) { + System.out.println("Errors occur when training in parallel! " + e); + } + + // Change sign to maximize and divide the values by size of dataset + double result = l.getLogli(); + int n = data.size(); + result = -(result / n); + return result; + } + + @Override + public double[] takeDerivative(double[] w) { + double[] derivatives = new double[w.length]; + for (int i = 0; i < w.length; i++) { + derivatives[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); + } + + Loglikelihood logli = new Loglikelihood(0, derivatives); // we dont care loglikelihood value + LogliComputer logliComp = new LogliComputer(w, featureGen, data, logli); + Scheduler sch = new Scheduler(logliComp, featureGen.getParams().getNumthreads(), Scheduler.DYNAMIC_NEXT_AVAILABLE); + try { + sch.run(); + } catch (Exception e) { + System.out.println("Errors occur when training in parallel! " + e); + } + + // Change sign to maximize and divide the values by size of dataset + int n = data.size(); + + double[] result = new double[w.length]; + System.arraycopy(logli.getDerivatives(), 0, result, 0, w.length); + for (int i = 0; i < result.length; i++) { + result[i] = -(result[i] / n); + } + + return result; + } + + @Override + public double[] takeDerivative(double[] w, int[] index) { + int l = w.length; + double[] result = new double[l]; + + for(int i : index){ + double[] eachDev = takeDerivative(w, i); + for (int j = 0; j < l; j++){ + result[j] += eachDev[j]; + } + } + + return result; + } + + @Override + public double[] takeDerivative(double[] w, int index) { + double[] result = new double[w.length]; + for (int i = 0; i < w.length; i++) { + result[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); + } + + Loglikelihood llh = new Loglikelihood(0, result); // can set loglikelihood any value; we dont care about this value + LogliComputer llc = new LogliComputer(w, featureGen, data, llh); + Loglikelihood l = (Loglikelihood) llc.compute(index); + + double[] d = l.getDerivatives(); + + for (int i = 0; i < w.length; i++) { + result[i] += d[i]; + } + + // Change sign to maximize and divide the values by size of dataset + int n = data.size(); + for (int i = 0; i < w.length; i++){ + result[i] = -(result[i] / n); + } + return result; + } +} \ No newline at end of file diff --git a/src/optimization/SquareLossFunction.java b/src/optimization/SquareLossFunction.java new file mode 100644 index 0000000..f4d4ab0 --- /dev/null +++ b/src/optimization/SquareLossFunction.java @@ -0,0 +1,101 @@ +package optimization; + +import java.util.ArrayList; + +/** + * implementation of square loss function. It is used to test optimization function + * P = \sum_{i=1}^n (w * x_i - y_i) ^ 2 / n + * where n is total number of data point + * @author tndoan + * + */ +public class SquareLossFunction extends AbstractSVRGFunction{ + ArrayList data; // data[i] = x_i + ArrayList response; // response[i] = y_i + + public SquareLossFunction(ArrayList array, double[] r) { + assert(array.size() == r.length); + + // init data + data = new ArrayList<>(); + for (double[] x : array){ + double[] d = new double[x.length]; + System.arraycopy(x, 0, d, 0, x.length); + data.add(d); + } + + // init response + response = new ArrayList<>(); + for (double y : r) + response.add(y); + } + + @Override + public int getNumberOfDataPoints(){ + return data.size(); + } + + @Override + public double valueAt(double[] w) { + int n = data.size(); // number of data points + double result = 0; + + for (int i = 0; i < n; i++){ + double[] point = data.get(i); + double y = response.get(i); + double phi = 0; // phi = w * point; w and point are vectors + for ( int j = 0; j < w.length; j++){ + phi += w[j] * point[j]; + } + + phi -= y; + result += phi * phi; + } + + result /= n; + + return result; + } + + @Override + public double[] takeDerivative(double[] w) { + double[] result = new double[w.length]; + int n = data.size(); // number of data points + + for (int i = 0; i < n; i++){ + double[] r = takeDerivative(w, i); + for (int j = 0; j < result.length; j++){ + result[j] += r[j]; + } + } + + return result; + } + + @Override + public double[] takeDerivative(double[] w, int[] index) { + // TODO Auto-generated method stub + return null; + } + + @Override + public double[] takeDerivative(double[] w, int index) { + double[] point = data.get(index); + double r = response.get(index); + + double[] result = new double[w.length]; + + double phi = 0.0; + for (int i = 0; i < w.length; i++){ + phi += w[i] * point[i]; + } + + phi -= r; + + for (int i = 0; i < w.length; i++){ + result[i] = phi * point[i] * 2; + } + + return result; + } +} From 3ed3362c237fce056b2bf5758ae8c12bbb32590d Mon Sep 17 00:00:00 2001 From: tndoan Date: Tue, 28 Jul 2015 14:38:31 +0800 Subject: [PATCH 09/12] add implementation of SVRG and SGD --- src/optimization/Minimizer.java | 67 +++++++++++++++++++++++++++ src/optimization/SGDMinimizer.java | 40 ++++++++++++++++ src/optimization/SVRGMinimizer.java | 72 +++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+) create mode 100644 src/optimization/Minimizer.java create mode 100644 src/optimization/SGDMinimizer.java create mode 100644 src/optimization/SVRGMinimizer.java diff --git a/src/optimization/Minimizer.java b/src/optimization/Minimizer.java new file mode 100644 index 0000000..7f18ed7 --- /dev/null +++ b/src/optimization/Minimizer.java @@ -0,0 +1,67 @@ +package optimization; + +/** + * abstract class for implementing first-order optimization + * @author tndoan + * + */ +public abstract class Minimizer { + + /** + * isConv = false if iteration terminates because of running more than max iterations; otherwise, true + */ + protected boolean isConv; + + /** + * after running algorithm, isConv = true if algorithm terminates because of convergence; + * otherwise(reach maximum number of iteration) isConv = false + * @return isConv + */ + public boolean isConv() { + return isConv; + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init){ + return minimize(f, init, 0.1); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate){ + return minimize(f, init, learningRate, 50); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @param maxPasses maximum number of data pass + * @return value that minimize function + */ + public double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate, int maxPasses){ + return minimize(f, init, learningRate, maxPasses, 0.001); + } + + /** + * finding value that minimize function via 1st-order optimization + * @param f function that we want to optimize + * @param init initial values + * @param learningRate learning rate of each iteration + * @param maxPasses maximum number of data pass + * @param funcTol threshold to stop iteration before reaching max number of iteration + * @return value that minimize function + */ + public abstract double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate, int maxPasses, double funcTol); +} \ No newline at end of file diff --git a/src/optimization/SGDMinimizer.java b/src/optimization/SGDMinimizer.java new file mode 100644 index 0000000..11b1cae --- /dev/null +++ b/src/optimization/SGDMinimizer.java @@ -0,0 +1,40 @@ +package optimization; + +/** + * implementation of Stochastic Gradient Descent + * @author tndoan + * + */ +public class SGDMinimizer extends Minimizer { + + @Override + public double[] minimize(AbstractSVRGFunction f, double[] init, + double learningRate, int maxPasses, double funcTol) { + + double[] result = new double[init.length]; + System.arraycopy(init, 0, result, 0, init.length); + double pre_obj = Double.MAX_VALUE; + + for (int i = 0; i < maxPasses; i++){ + for (int j = 0; j < f.getNumberOfDataPoints(); j++){ + + double[] dev = f.takeDerivative(result, j); + for (int k = 0; k < dev.length; k++){ + result[k] -= learningRate * dev[k]; + } +// System.out.println(f.valueAt(result)); + } + + // check convergence + double cur_obj = f.valueAt(result); + if (i > 1 && Math.abs(pre_obj - cur_obj) < funcTol) { + this.isConv = true; + break; + } + pre_obj = cur_obj; + System.out.println(f.valueAt(result)); + } + + return result; + } +} diff --git a/src/optimization/SVRGMinimizer.java b/src/optimization/SVRGMinimizer.java new file mode 100644 index 0000000..c3be31f --- /dev/null +++ b/src/optimization/SVRGMinimizer.java @@ -0,0 +1,72 @@ +package optimization; + +import java.util.Random; + +/** + * It is the implementation of SVRG algorithm + * http://stat.rutgers.edu/home/tzhang/papers/nips13-svrg.pdf + * @author tndoan + * + */ +public class SVRGMinimizer extends Minimizer { + + @Override + public double[] minimize(AbstractSVRGFunction f, double[] init, + double learningRate, int maxPasses, double funcTol) { + int n = f.getNumberOfDataPoints(); + int upFreq = 2 * n; + int d = init.length; + + double[] w_0_tilde = new double[d]; + double[] mu_tilde = new double[d]; + double[] w_tilde = new double[d]; + double[] w_0 = new double[d]; + + Random rand = new Random(); + + // copy init value + System.arraycopy(init, 0, w_0_tilde, 0, d); + double pre_obj = Double.MAX_VALUE; + + for (int s = 0; s < maxPasses; s++){ + // w_tilde = w_0_tilde + System.arraycopy(w_0_tilde, 0, w_tilde, 0, d); + + // mu_tilde = (\sum_{i=1}^n \delta \psi_i (w_tilde)) / n + System.arraycopy(f.takeDerivative(w_tilde), 0, mu_tilde, 0, d); + for (int i = 0; i < d; i++) + mu_tilde[i] /= (double) n; + + //w_0 = w_tilde + System.arraycopy(w_tilde, 0, w_0, 0, d); + + for (int t = 0; t < upFreq; t++){ +// System.out.println("t:" +t); + int i_t = rand.nextInt(n); + double[] f1 = f.takeDerivative(w_0, i_t); + double[] f2 = f.takeDerivative(w_tilde, i_t); + + for (int i = 0; i < d; i++){ + w_0[i] = w_0[i] - learningRate * (f1[i] - f2[i] + mu_tilde[i]); + } + } + + // w_0_tilde = w_0 + System.arraycopy(w_0, 0, w_0_tilde, 0, d); + + // check convergence + double curr_obj = f.valueAt(w_0_tilde); + + double diff = Math.abs(curr_obj - pre_obj); + if (s > 1 && diff < funcTol) { + // s > 1 to ensure that objective function is calculated at least 1 time. + this.isConv = true; + break; + } + System.out.println("Objective function pre:" + pre_obj + " curr:" + curr_obj + " funcTol:" + funcTol); + pre_obj = curr_obj; + } + + return w_0_tilde; + } +} From 2a6f477f134a1796b9572d045406408ccb208eb8 Mon Sep 17 00:00:00 2001 From: tndoan Date: Sat, 8 Aug 2015 00:42:58 +0800 Subject: [PATCH 10/12] organize code, add and use derivative at each data point for SVRG --- src/optimization/AbstractSVRGFunction.java | 16 +-- src/optimization/FirstOrderDiffFunction.java | 120 +++++++++---------- src/optimization/SVRGMinimizer.java | 26 ++-- src/optimization/SquareLossFunction.java | 6 + 4 files changed, 90 insertions(+), 78 deletions(-) diff --git a/src/optimization/AbstractSVRGFunction.java b/src/optimization/AbstractSVRGFunction.java index 48a97b6..6a12cc8 100644 --- a/src/optimization/AbstractSVRGFunction.java +++ b/src/optimization/AbstractSVRGFunction.java @@ -1,8 +1,5 @@ package optimization; -import java.util.ArrayList; - -import HOSemiCRF.DataSequence; /** * It is the abstract of function @@ -12,15 +9,11 @@ public abstract class AbstractSVRGFunction { // similar AbstractStochasticCachingDiffUpdateFunction - protected ArrayList data; - /** * * @return the number of data points */ - public int getNumberOfDataPoints(){ - return data.size(); - } + public abstract int getNumberOfDataPoints(); /** * Get value of function @@ -51,4 +44,11 @@ public int getNumberOfDataPoints(){ * @return derivative of function with parameter w at data point whose index is specified */ public abstract double[] takeDerivative(double[] w, int index); + + /** + * take derivative of function of each data point + * @param w value of parameter + * @return 2d array derivative of function with parameter w at each data point. result[i] is the derivative of data point i. + */ + public abstract double[][] takeEachDerivative(double[] w); } \ No newline at end of file diff --git a/src/optimization/FirstOrderDiffFunction.java b/src/optimization/FirstOrderDiffFunction.java index 335cc3f..8a06665 100644 --- a/src/optimization/FirstOrderDiffFunction.java +++ b/src/optimization/FirstOrderDiffFunction.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import HOSemiCRF.DataSequence; +import HOSemiCRF.ExtLogliComputer; import HOSemiCRF.FeatureGenerator; import HOSemiCRF.LogliComputer; import HOSemiCRF.Loglikelihood; @@ -10,26 +11,36 @@ public class FirstOrderDiffFunction extends AbstractSVRGFunction { - /** - * Feature Generator - */ FeatureGenerator featureGen; // Feature generator + ArrayList data; + + // Stored results + private Loglikelihood logli; + private double[][] eachDerivatives; public FirstOrderDiffFunction(FeatureGenerator fg, ArrayList data) { this.data = data; this.featureGen = fg; } - @Override - public double valueAt(double[] w) { - double logli = 0; - for (int i = 0; i < w.length; i++) { - logli -= ((w[i] * w[i]) * featureGen.getParams().getInvSigmaSquare()) / 2; - } - - Loglikelihood l = new Loglikelihood(logli, new double[w.length]); // we dont care about derivative - - LogliComputer logliComp = new LogliComputer(w, featureGen, data, l); + /** + * Compute loglikelihood, derivatives for all data, and derivatives at each data point + * Store in eachDerivatives + * @param w value of parameters + */ + public void computeAllValues(double[] w) { + // init eachDerivative + logli = new Loglikelihood(w.length); + eachDerivatives = new double[data.size()][w.length]; + for (int i = 0; i < w.length; i++){ + for (int j = 0; j < data.size(); j++){ + eachDerivatives[j][i] = -(w[i] * featureGen.getParams().getInvSigmaSquare()); + } + logli.logli -= ((w[i] * w[i]) * featureGen.getParams().getInvSigmaSquare()) / 2; + logli.derivatives[i] = -(w[i] * featureGen.getParams().getInvSigmaSquare()); + } + + ExtLogliComputer logliComp = new ExtLogliComputer(w, featureGen, data, logli, eachDerivatives); Scheduler sch = new Scheduler(logliComp, featureGen.getParams().getNumthreads(), Scheduler.DYNAMIC_NEXT_AVAILABLE); try { sch.run(); @@ -37,39 +48,34 @@ public double valueAt(double[] w) { System.out.println("Errors occur when training in parallel! " + e); } - // Change sign to maximize and divide the values by size of dataset - double result = l.getLogli(); - int n = data.size(); - result = -(result / n); - return result; + // Change sign to maximize + for (int i = 0; i < w.length; i++) { + logli.derivatives[i] = -logli.derivatives[i] / data.size(); + for (int j = 0; j < data.size(); j++){ + eachDerivatives[j][i] = -eachDerivatives[j][i] / data.size(); + } + } + logli.logli = -logli.logli / data.size(); + } + + /** + * Ensure that computeAllValues(w) was called before this + */ + @Override + public double valueAt(double[] w) { + return logli.getLogli(); } @Override + // Ensure that computeAllValues(w) was called before this public double[] takeDerivative(double[] w) { - double[] derivatives = new double[w.length]; - for (int i = 0; i < w.length; i++) { - derivatives[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); - } - - Loglikelihood logli = new Loglikelihood(0, derivatives); // we dont care loglikelihood value - LogliComputer logliComp = new LogliComputer(w, featureGen, data, logli); - Scheduler sch = new Scheduler(logliComp, featureGen.getParams().getNumthreads(), Scheduler.DYNAMIC_NEXT_AVAILABLE); - try { - sch.run(); - } catch (Exception e) { - System.out.println("Errors occur when training in parallel! " + e); - } - - // Change sign to maximize and divide the values by size of dataset - int n = data.size(); - - double[] result = new double[w.length]; - System.arraycopy(logli.getDerivatives(), 0, result, 0, w.length); - for (int i = 0; i < result.length; i++) { - result[i] = -(result[i] / n); - } - - return result; + return logli.getDerivatives(); + } + + @Override + // Ensure that computeAllValues(w) was called before this + public double[][] takeEachDerivative(double[] w){ + return eachDerivatives; } @Override @@ -88,27 +94,21 @@ public double[] takeDerivative(double[] w, int[] index) { } @Override - public double[] takeDerivative(double[] w, int index) { - double[] result = new double[w.length]; - for (int i = 0; i < w.length; i++) { - result[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); - } - - Loglikelihood llh = new Loglikelihood(0, result); // can set loglikelihood any value; we dont care about this value + public double[] takeDerivative(double[] w, int index) { + Loglikelihood llh = new Loglikelihood(w.length); // can set loglikelihood any value; we dont care about this value LogliComputer llc = new LogliComputer(w, featureGen, data, llh); - Loglikelihood l = (Loglikelihood) llc.compute(index); - - double[] d = l.getDerivatives(); + double[] result = ((Loglikelihood) llc.compute(index)).getDerivatives(); for (int i = 0; i < w.length; i++) { - result[i] += d[i]; - } - - // Change sign to maximize and divide the values by size of dataset - int n = data.size(); - for (int i = 0; i < w.length; i++){ - result[i] = -(result[i] / n); - } + result[i] -= (w[i] * featureGen.getParams().getInvSigmaSquare()); + result[i] = -result[i] / data.size(); // Change sign to maximize + } + return result; } + + @Override + public int getNumberOfDataPoints() { + return data.size(); + } } \ No newline at end of file diff --git a/src/optimization/SVRGMinimizer.java b/src/optimization/SVRGMinimizer.java index c3be31f..ce90b88 100644 --- a/src/optimization/SVRGMinimizer.java +++ b/src/optimization/SVRGMinimizer.java @@ -14,38 +14,41 @@ public class SVRGMinimizer extends Minimizer { public double[] minimize(AbstractSVRGFunction f, double[] init, double learningRate, int maxPasses, double funcTol) { int n = f.getNumberOfDataPoints(); - int upFreq = 2 * n; + int upFreq = 10; int d = init.length; double[] w_0_tilde = new double[d]; - double[] mu_tilde = new double[d]; + double[] mu_tilde; // = new double[d]; double[] w_tilde = new double[d]; double[] w_0 = new double[d]; - Random rand = new Random(); + Random rand = new Random(123456987); // copy init value System.arraycopy(init, 0, w_0_tilde, 0, d); - double pre_obj = Double.MAX_VALUE; + + ((FirstOrderDiffFunction)f).computeAllValues(w_0_tilde); + double pre_obj = f.valueAt(w_0_tilde); + double[][] eachDerivatives = f.takeEachDerivative(w_0_tilde); + mu_tilde = f.takeDerivative(w_0_tilde); for (int s = 0; s < maxPasses; s++){ // w_tilde = w_0_tilde System.arraycopy(w_0_tilde, 0, w_tilde, 0, d); - // mu_tilde = (\sum_{i=1}^n \delta \psi_i (w_tilde)) / n - System.arraycopy(f.takeDerivative(w_tilde), 0, mu_tilde, 0, d); - for (int i = 0; i < d; i++) + // mu_tilde = (\sum_{i=1}^n \delta \psi_i (w_tilde)) / n + for (int i = 0; i < d; i++){ mu_tilde[i] /= (double) n; + } //w_0 = w_tilde System.arraycopy(w_tilde, 0, w_0, 0, d); for (int t = 0; t < upFreq; t++){ -// System.out.println("t:" +t); int i_t = rand.nextInt(n); double[] f1 = f.takeDerivative(w_0, i_t); - double[] f2 = f.takeDerivative(w_tilde, i_t); - + double[] f2 = eachDerivatives[i_t]; + for (int i = 0; i < d; i++){ w_0[i] = w_0[i] - learningRate * (f1[i] - f2[i] + mu_tilde[i]); } @@ -55,7 +58,10 @@ public double[] minimize(AbstractSVRGFunction f, double[] init, System.arraycopy(w_0, 0, w_0_tilde, 0, d); // check convergence + ((FirstOrderDiffFunction)f).computeAllValues(w_0_tilde); double curr_obj = f.valueAt(w_0_tilde); + mu_tilde = f.takeDerivative(w_0_tilde); + eachDerivatives = f.takeEachDerivative(w_0_tilde); double diff = Math.abs(curr_obj - pre_obj); if (s > 1 && diff < funcTol) { diff --git a/src/optimization/SquareLossFunction.java b/src/optimization/SquareLossFunction.java index f4d4ab0..bc91a52 100644 --- a/src/optimization/SquareLossFunction.java +++ b/src/optimization/SquareLossFunction.java @@ -98,4 +98,10 @@ public double[] takeDerivative(double[] w, int index) { return result; } + + @Override + public double[][] takeEachDerivative(double[] w) { + // TODO Auto-generated method stub + return null; + } } From 58dfb3330102661af57ba7ab23957f4f45d0bef4 Mon Sep 17 00:00:00 2001 From: tndoan Date: Sat, 8 Aug 2015 00:43:51 +0800 Subject: [PATCH 11/12] change logli and derivatives to public to access --- src/HOSemiCRF/Loglikelihood.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/HOSemiCRF/Loglikelihood.java b/src/HOSemiCRF/Loglikelihood.java index d7fa82a..1e5e18e 100644 --- a/src/HOSemiCRF/Loglikelihood.java +++ b/src/HOSemiCRF/Loglikelihood.java @@ -26,8 +26,8 @@ */ public class Loglikelihood { - double logli; // Loglikelihood value - double derivatives[]; // Loglikelihood derivatives + public double logli; // Loglikelihood value + public double derivatives[]; // Loglikelihood derivatives /** * Construct a loglikelihood with a given number of features. From e31d7926c8a70c0c93f474c0ac06d3536ae695e2 Mon Sep 17 00:00:00 2001 From: tndoan Date: Sat, 8 Aug 2015 00:44:41 +0800 Subject: [PATCH 12/12] extends LogliComputer to do parallel and get derivatives of each data point --- src/HOSemiCRF/ExtLogliComputer.java | 46 +++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 src/HOSemiCRF/ExtLogliComputer.java diff --git a/src/HOSemiCRF/ExtLogliComputer.java b/src/HOSemiCRF/ExtLogliComputer.java new file mode 100644 index 0000000..b5a0300 --- /dev/null +++ b/src/HOSemiCRF/ExtLogliComputer.java @@ -0,0 +1,46 @@ +package HOSemiCRF; + +import java.util.ArrayList; + +/** + * This class extends LogliComputer in order to do parallel and get the derivative of each data point. + * @author tndoan + * + */ +public class ExtLogliComputer extends LogliComputer { + + /** + * store derivative of each data points + */ + private double[][] eachDerivatives; + + public ExtLogliComputer(double[] lambdaValues, FeatureGenerator fgen, + ArrayList td, Loglikelihood loglh, double[][] eachDerivatives) { + super(lambdaValues, fgen, td, loglh); + this.eachDerivatives = eachDerivatives; + } + + /** + * Override compute method of super class to store each derivative to {@link ExtLogliComputer#eachDerivatives} + */ + @Override + public Object compute(int taskID) { + Object result = super.compute(taskID); + double[] d = ((Loglikelihood) result).derivatives; + + for(int i = 0; i < d.length; i++){ + eachDerivatives[taskID][i] += d[i]; + } + + return result; + } + + /** + * get the derivative of each data point + * @return 2d array result. result[i] is derivative of data point i. + */ + public double[][] getEachDerivative() { + return eachDerivatives; + } + +} \ No newline at end of file