Examples with NETaggerLevel2 edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2 used on opensource projects

Example 6 with NETaggerLevel2

use of edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2 in project cogcomp-nlp by CogComp.

the class NerTagger method main.

public static void main(String[] args) {
    if (args.length == 0) {
        printUsage(System.out);
        System.exit(-1);
    }
    ParametersForLbjCode cp = null;
    try {
        boolean areWeTraining = args[0].equalsIgnoreCase("-train");
        ResourceManager rm = new ResourceManager(args[args.length - 1]);
        cp = Parameters.readConfigAndLoadExternalData(args[args.length - 1], areWeTraining);
        if (args[0].equalsIgnoreCase("-train")) {
            String dataFormat;
            // config file is always the last one.
            if (args.length < 5) {
                dataFormat = "-c";
            } else {
                dataFormat = args[3];
            }
            LearningCurveMultiDataset.getLearningCurve(-1, dataFormat, args[1], args[2], false, cp);
        } else if (args[0].equalsIgnoreCase("-trainFixedIterations"))
            LearningCurveMultiDataset.getLearningCurve(Integer.parseInt(args[1]), args[2], args[3], false, cp);
        else {
            // load up the models
            ModelLoader.load(rm, rm.getString("modelName"), false, cp);
            if (args[0].equalsIgnoreCase("-annotate")) {
                String dataFormat;
                // config file is always the last one.
                if (args.length < 5) {
                    dataFormat = "-plaintext";
                } else {
                    dataFormat = args[3];
                }
                NETagPlain.tagData(args[1], args[2], dataFormat, cp);
            }
            if (args[0].equalsIgnoreCase("-demo")) {
                String input = "";
                while (!input.equalsIgnoreCase("quit")) {
                    input = Keyboard.readLine();
                    if (input.equalsIgnoreCase("quit"))
                        System.exit(0);
                    String res = NETagPlain.tagLine(input, (NETaggerLevel1) cp.taggerLevel1, (NETaggerLevel2) cp.taggerLevel2, cp);
                    res = NETagPlain.insertHtmlColors(res);
                    StringTokenizer st = new StringTokenizer(res);
                    StringBuilder output = new StringBuilder();
                    while (st.hasMoreTokens()) {
                        String s = st.nextToken();
                        output.append(" ").append(s);
                    }
                    logger.info(output.toString());
                }
            }
            if (args[0].equalsIgnoreCase("-test")) {
                String dataFormat;
                // config file is always the last one.
                if (args.length < 4) {
                    dataFormat = "-c";
                } else {
                    dataFormat = args[2];
                }
                NETesterMultiDataset.test(args[1], true, dataFormat, cp.labelsToIgnoreInEvaluation, cp.labelsToAnonymizeInEvaluation, cp);
            }
            if (args[0].equalsIgnoreCase("-dumpFeatures"))
                NETesterMultiDataset.dumpFeaturesLabeledData(args[1], args[2], cp);
        }
    } catch (Exception e) {
        logger.error("Exception caught: ");
        e.printStackTrace();
        logger.error("");
        printUsage(System.err);
    }
}

Example 7 with NETaggerLevel2

use of edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2 in project cogcomp-nlp by CogComp.

the class LearningCurveMultiDataset method getLearningCurve.

/**
 * use fixedNumIterations=-1 if you want to use the automatic convergence criterion, incremental
 * true will start with the existing models weights, and continue training with that set of default
 * weights. Training data is assumed to be in column format.
 * <p>
 * @param fixedNumIterations if this is > -1 the number of training iterations that will run.
 * @param trainDataSet the path on the file system for the training data.
 * @param testDataSet the path on the file system for the test data used to test convergence.
 * @param incremental if the model is being incremented, this is true.
 * @throws Exception
 */
public static void getLearningCurve(Vector<Data> trainDataSet, Vector<Data> testDataSet, int fixedNumIterations, boolean incremental, ParametersForLbjCode params) throws Exception {
    double bestF1Level1 = -1;
    int bestRoundLevel1 = 0;
    // Get the directory name (<configname>.model is appended in LbjTagger/Parameters.java:139)
    String modelPath = params.pathToModelFile;
    String modelPathDir = modelPath.substring(0, modelPath.lastIndexOf("/"));
    if (IOUtils.exists(modelPathDir)) {
        if (!IOUtils.isDirectory(modelPathDir)) {
            String msg = "ERROR: " + NAME + ".getLearningCurve(): model directory '" + modelPathDir + "' already exists as a (non-directory) file.";
            logger.error(msg);
            throw new IOException(msg);
        } else
            logger.warn(NAME + ".getLearningCurve(): writing to existing model path '" + modelPathDir + "'...");
    } else {
        IOUtils.mkdir(modelPathDir);
    }
    NETaggerLevel1.Parameters paramLevel1 = new NETaggerLevel1.Parameters();
    paramLevel1.baseLTU = new SparseAveragedPerceptron(params.learningRatePredictionsLevel1, 0, params.thicknessPredictionsLevel1);
    paramLevel1.baseLTU.featurePruningThreshold = params.featurePruningThreshold;
    logger.info("Level 1 classifier learning rate = " + params.learningRatePredictionsLevel1 + ", thickness = " + params.thicknessPredictionsLevel1);
    NETaggerLevel1 tagger1 = new NETaggerLevel1(paramLevel1, modelPath + ".level1", modelPath + ".level1.lex");
    if (!incremental) {
        logger.info("Training L1 model from scratch.");
        tagger1.forget();
    } else {
        logger.info("Training L1 model incrementally.");
    }
    params.taggerLevel1 = tagger1;
    for (int dataId = 0; dataId < trainDataSet.size(); dataId++) {
        Data trainData = trainDataSet.elementAt(dataId);
        if (params.featuresToUse.containsKey("PredictionsLevel1")) {
            PredictionsAndEntitiesConfidenceScores.getAndMarkEntities(trainData, NEWord.LabelToLookAt.GoldLabel);
            TwoLayerPredictionAggregationFeatures.setLevel1AggregationFeatures(trainData, true);
        }
    }
    // preextract the L1 test and train data.
    String path = params.pathToModelFile;
    String trainPathL1 = path + ".level1.prefetchedTrainData";
    File deleteme = new File(trainPathL1);
    if (deleteme.exists())
        deleteme.delete();
    String testPathL1 = path + ".level1.prefetchedTestData";
    deleteme = new File(testPathL1);
    if (deleteme.exists())
        deleteme.delete();
    logger.info("Pre-extracting the training data for Level 1 classifier, saving to " + trainPathL1);
    BatchTrainer bt1train = prefetchAndGetBatchTrainer(tagger1, trainDataSet, trainPathL1, params);
    logger.info("Pre-extracting the testing data for Level 1 classifier, saving to " + testPathL1);
    BatchTrainer bt1test = prefetchAndGetBatchTrainer(tagger1, testDataSet, testPathL1, params);
    Parser testParser1 = bt1test.getParser();
    // create the best model possible.
    {
        NETaggerLevel1 saveme = null;
        for (int i = 0; (fixedNumIterations == -1 && i < 200 && i - bestRoundLevel1 < 10) || (fixedNumIterations > 0 && i <= fixedNumIterations); ++i) {
            bt1train.train(1);
            testParser1.reset();
            TestDiscrete simpleTest = new TestDiscrete();
            simpleTest.addNull("O");
            TestDiscrete.testDiscrete(simpleTest, tagger1, null, testParser1, true, 0);
            double f1Level1 = simpleTest.getOverallStats()[2];
            if (Double.isNaN(f1Level1))
                f1Level1 = 0;
            if (f1Level1 > bestF1Level1) {
                bestF1Level1 = f1Level1;
                bestRoundLevel1 = i;
                saveme = (NETaggerLevel1) tagger1.clone();
                saveme.beginTraining();
                System.out.println(saveme);
                System.out.println(bestF1Level1);
                System.out.println(f1Level1);
            }
            logger.info(i + " rounds.  Best so far for Level1 : (" + bestRoundLevel1 + ")=" + bestF1Level1);
        }
        saveme.getBaseLTU().featurePruningThreshold = params.featurePruningThreshold;
        saveme.doneTraining();
        saveme.save();
        logger.info("Level 1; best round : " + bestRoundLevel1 + "\tbest F1 : " + bestF1Level1);
    }
    // Read the best model back in, optimize by pruning useless features, then write it agains
    tagger1 = new NETaggerLevel1(paramLevel1, modelPath + ".level1", modelPath + ".level1.lex");
    // trash the l2 prefetch data
    String trainPathL2 = path + ".level2.prefetchedTrainData";
    deleteme = new File(trainPathL2);
    if (deleteme.exists())
        deleteme.delete();
    String testPathL2 = path + ".level2.prefetchedTestData";
    deleteme = new File(testPathL1);
    if (deleteme.exists())
        deleteme.delete();
    NETaggerLevel2.Parameters paramLevel2 = new NETaggerLevel2.Parameters();
    paramLevel2.baseLTU = new SparseAveragedPerceptron(params.learningRatePredictionsLevel2, 0, params.thicknessPredictionsLevel2);
    paramLevel2.baseLTU.featurePruningThreshold = params.featurePruningThreshold;
    NETaggerLevel2 tagger2 = new NETaggerLevel2(paramLevel2, params.pathToModelFile + ".level2", params.pathToModelFile + ".level2.lex");
    if (!incremental) {
        logger.info("Training L2 model from scratch.");
        tagger2.forget();
    } else {
        logger.info("Training L2 model incrementally.");
    }
    params.taggerLevel2 = tagger2;
    // Previously checked if PatternFeatures was in featuresToUse.
    if (params.featuresToUse.containsKey("PredictionsLevel1")) {
        logger.info("Level 2 classifier learning rate = " + params.learningRatePredictionsLevel2 + ", thickness = " + params.thicknessPredictionsLevel2);
        double bestF1Level2 = -1;
        int bestRoundLevel2 = 0;
        logger.info("Pre-extracting the training data for Level 2 classifier, saving to " + trainPathL2);
        BatchTrainer bt2train = prefetchAndGetBatchTrainer(tagger2, trainDataSet, trainPathL2, params);
        logger.info("Pre-extracting the testing data for Level 2 classifier, saving to " + testPathL2);
        BatchTrainer bt2test = prefetchAndGetBatchTrainer(tagger2, testDataSet, testPathL2, params);
        Parser testParser2 = bt2test.getParser();
        // create the best model possible.
        {
            NETaggerLevel2 saveme = null;
            for (int i = 0; (fixedNumIterations == -1 && i < 200 && i - bestRoundLevel2 < 10) || (fixedNumIterations > 0 && i <= fixedNumIterations); ++i) {
                logger.info("Learning level 2 classifier; round " + i);
                bt2train.train(1);
                logger.info("Testing level 2 classifier;  on prefetched data, round: " + i);
                testParser2.reset();
                TestDiscrete simpleTest = new TestDiscrete();
                simpleTest.addNull("O");
                TestDiscrete.testDiscrete(simpleTest, tagger2, null, testParser2, true, 0);
                double f1Level2 = simpleTest.getOverallStats()[2];
                if (f1Level2 >= bestF1Level2) {
                    bestF1Level2 = f1Level2;
                    bestRoundLevel2 = i;
                    saveme = (NETaggerLevel2) tagger2.clone();
                    saveme.beginTraining();
                }
                logger.info(i + " rounds.  Best so far for Level2 : (" + bestRoundLevel2 + ") " + bestF1Level2);
            }
            saveme.getBaseLTU().featurePruningThreshold = params.featurePruningThreshold;
            saveme.doneTraining();
            saveme.save();
        }
        // trash the l2 prefetch data
        deleteme = new File(trainPathL2);
        if (deleteme.exists())
            deleteme.delete();
        deleteme = new File(testPathL1);
        if (deleteme.exists())
            deleteme.delete();
        logger.info("Level1: bestround=" + bestRoundLevel1 + "\t F1=" + bestF1Level1 + "\t Level2: bestround=" + bestRoundLevel2 + "\t F1=" + bestF1Level2);
    }
    NETesterMultiDataset.printTestResultsByDataset(testDataSet, tagger1, tagger2, true, params);
    /*
         * This will override the models forcing to save the iteration we're interested in- the
         * fixedNumIterations iteration, the last one. But note - both layers will be saved for this
         * iteration. If the best performance for one of the layers came before the final iteration,
         * we're in a small trouble- the performance will decrease
         */
    if (fixedNumIterations > -1) {
        tagger1.save();
        tagger2.save();
    }
}

Example 8 with NETaggerLevel2

use of edu.illinois.cs.cogcomp.ner.LbjFeatures.NETaggerLevel2 in project cogcomp-nlp by CogComp.

the class NerBenchmark method trainModel.

/**
 * This method does all the work of actually training the model. It requires the config file name, the name of the training
 * directory, and a file representation of that, and the same for the dev and test directories. The model is trained only
 * against the training data, dev data is used to test for convergence, rather than using the number of iterations, and the
 * test data is held out to compute the final accuracy results which are returned.
 * @param confFile the name of the config file.
 * @param trainDirName the name of the train directory.
 * @param trainDir the file object for the train directory.
 * @param devDirName the name of the dev directory.
 * @param devDir the file object for the dev directory.
 * @param testDirName the test directory name.
 * @param testDir the test directory file object.
 * @return
 * @throws Exception
 */
private Vector<TestDiscrete[]> trainModel(String confFile, String trainDirName, File trainDir, String devDirName, File devDir, String testDirName, File testDir) throws Exception {
    System.out.println("\n\n----- Training models for evaluation for " + confFile + " ------");
    ParametersForLbjCode prms = Parameters.readConfigAndLoadExternalData(confFile, true);
    ResourceManager rm = new ResourceManager(confFile);
    ModelLoader.load(rm, rm.getString("modelName"), true, prms);
    NETaggerLevel1 taggerLevel1 = (NETaggerLevel1) prms.taggerLevel1;
    NETaggerLevel2 taggerLevel2 = (NETaggerLevel2) prms.taggerLevel2;
    SparseAveragedPerceptron sap1 = (SparseAveragedPerceptron) taggerLevel1.getBaseLTU();
    sap1.setLearningRate(prms.learningRatePredictionsLevel1);
    sap1.setThickness(prms.thicknessPredictionsLevel1);
    System.out.println("L1 learning rate = " + sap1.getLearningRate() + ", thickness = " + sap1.getPositiveThickness());
    if (prms.featuresToUse.containsKey("PredictionsLevel1")) {
        SparseAveragedPerceptron sap2 = (SparseAveragedPerceptron) taggerLevel2.getBaseLTU();
        sap2.setLearningRate(prms.learningRatePredictionsLevel2);
        sap2.setThickness(prms.thicknessPredictionsLevel2);
        System.out.println("L2 learning rate = " + sap2.getLearningRate() + ", thickness = " + sap2.getPositiveThickness());
    }
    // there is a training directory, with training enabled, so train. We use the same dataset
    // for both training and evaluating.
    LearningCurveMultiDataset.getLearningCurve(iterations, trainDirName, devDirName, incremental, prms);
    System.out.println("\n\n----- Final results for " + confFile + ", verbose ------");
    NETesterMultiDataset.test(testDirName, true, prms.labelsToIgnoreInEvaluation, prms.labelsToAnonymizeInEvaluation, prms);
    System.out.println("\n\n----- Final results for " + confFile + ", F1 only ------");
    return NETesterMultiDataset.test(testDirName, false, prms.labelsToIgnoreInEvaluation, prms.labelsToAnonymizeInEvaluation, prms);
}