Examples with MLMeasures edu.neu.ccs.pyramid.eval.MLMeasures used on opensource projects

Example 21 with MLMeasures

use of edu.neu.ccs.pyramid.eval.MLMeasures in project pyramid by cheng-li.

the class NoiseOptimizerTest method test1.

private static void test1() {
    MultiLabelClfDataSet train = MultiLabelSynthesizer.crfArgmaxDrop();
    MultiLabelClfDataSet test = MultiLabelSynthesizer.crfArgmax();
    TRECFormat.save(train, new File(TMP, "train"));
    TRECFormat.save(test, new File(TMP, "test"));
    CMLCRF cmlcrf = new CMLCRF(train);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(0, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(1, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(0, -10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(1, -10);
    MLScorer accScorer = new AccScorer();
    SubsetAccPredictor plugInAcc = new SubsetAccPredictor(cmlcrf);
    InstanceF1Predictor plugInF1 = new InstanceF1Predictor(cmlcrf);
    System.out.println(cmlcrf);
    System.out.println("training performance acc");
    System.out.println(new MLMeasures(cmlcrf, train));
    System.out.println("test performance acc");
    System.out.println(new MLMeasures(cmlcrf, test));
    System.out.println("training performance f1");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test performance f1");
    System.out.println(new MLMeasures(plugInF1, test));
    LogRiskOptimizer accOptimizer = new LogRiskOptimizer(train, accScorer, cmlcrf, 1, false, false, 1, 1);
    accOptimizer.iterate();
    System.out.println("after ML estimation");
    System.out.println("training with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, train));
    System.out.println("training with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, test));
    System.out.println("test with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, test));
    System.out.println(cmlcrf);
    NoiseOptimizer noiseOptimizer = new NoiseOptimizer(train, cmlcrf, 1);
    while (!noiseOptimizer.getTerminator().shouldTerminate()) {
        System.out.println("------------");
        noiseOptimizer.iterate();
        System.out.println(noiseOptimizer.objectiveDetail());
        System.out.println("training performance acc");
        System.out.println(new MLMeasures(cmlcrf, train));
        System.out.println("test performance acc");
        System.out.println(new MLMeasures(cmlcrf, test));
        System.out.println("training performance f1");
        System.out.println(new MLMeasures(plugInF1, train));
        System.out.println("test performance f1");
        System.out.println(new MLMeasures(plugInF1, test));
        System.out.println(cmlcrf);
    }
}

Example 22 with MLMeasures

use of edu.neu.ccs.pyramid.eval.MLMeasures in project pyramid by cheng-li.

the class CBMSOptimizerTest method test1.

private static void test1() throws Exception {
    MultiLabelClfDataSet dataSet = TRECFormat.loadMultiLabelClfDataSet(new File(DATASETS, "scene/train"), DataSetType.ML_CLF_DENSE, true);
    MultiLabelClfDataSet testSet = TRECFormat.loadMultiLabelClfDataSet(new File(DATASETS, "scene/test"), DataSetType.ML_CLF_DENSE, true);
    int numComponents = 50;
    CBMS cbms = new CBMS(dataSet.getNumClasses(), numComponents, dataSet.getNumFeatures());
    Set<MultiLabel> seen = DataSetUtil.gatherMultiLabels(dataSet).stream().collect(Collectors.toSet());
    MultiLabel empty = new MultiLabel();
    if (seen.contains(empty)) {
        cbms.setAllowEmpty(true);
        System.out.println("training set contains empty labels, automatically set predict.allowEmpty = true");
    } else {
        cbms.setAllowEmpty(false);
        System.out.println("training set does not contain empty labels, automatically set predict.allowEmpty = false");
    }
    CBMSOptimizer optimizer = new CBMSOptimizer(cbms, dataSet);
    optimizer.setPriorVarianceBinary(1);
    optimizer.setPriorVarianceMultiClass(1);
    optimizer.setComponentWeightsVariance(10);
    CBMSInitializer.initialize(cbms, dataSet, optimizer);
    System.out.println("after initialization");
    System.out.println("training performance");
    System.out.println(new MLMeasures(cbms, dataSet));
    System.out.println("test performance");
    System.out.println(new MLMeasures(cbms, testSet));
    for (int i = 1; i <= 50; i++) {
        System.out.println("iteration " + i);
        optimizer.eStep();
        System.out.println("after E");
        System.out.println("objective = " + optimizer.getObjective());
        System.out.println("multi-class obj = " + optimizer.multiClassClassifierObj());
        System.out.println("binary obj = " + optimizer.binaryObj());
        optimizer.mStep();
        System.out.println("after M");
        System.out.println("objective = " + optimizer.getObjective());
        System.out.println("multi-class obj = " + optimizer.multiClassClassifierObj());
        System.out.println("binary obj = " + optimizer.binaryObj());
        //            System.out.println(Arrays.toString(optimizer.gammas[0]));
        //            System.out.println(Arrays.toString(cbms.predictClassProbs(dataSet.getRow(0))));
        //            System.out.println(cbms.getBinaryClassifiers()[0]);
        //            System.out.println(cbms.predictByMarginals(dataSet.getRow(0)));
        //            System.out.println(cbms.predict(dataSet.getRow(199)));
        System.out.println("training performance");
        System.out.println(new MLMeasures(cbms, dataSet));
        System.out.println("test performance");
        System.out.println(new MLMeasures(cbms, testSet));
    }
}

Example 23 with MLMeasures

use of edu.neu.ccs.pyramid.eval.MLMeasures in project pyramid by cheng-li.

the class LogRiskOptimizerTest method test3.

private static void test3() {
    MultiLabelClfDataSet train = MultiLabelSynthesizer.crfSample();
    MultiLabelClfDataSet test = MultiLabelSynthesizer.crfSample();
    CMLCRF cmlcrf = new CMLCRF(train);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(0, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(1, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(0, -10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(1, -10);
    MLScorer fScorer = new FScorer();
    MLScorer accScorer = new AccScorer();
    SubsetAccPredictor plugInAcc = new SubsetAccPredictor(cmlcrf);
    LogRiskOptimizer fOptimizer = new LogRiskOptimizer(train, fScorer, cmlcrf, 1, false, false, 1, 1);
    InstanceF1Predictor plugInF1 = new InstanceF1Predictor(cmlcrf);
    System.out.println(cmlcrf);
    System.out.println("training performance acc");
    System.out.println(new MLMeasures(cmlcrf, train));
    System.out.println("test performance acc");
    System.out.println(new MLMeasures(cmlcrf, test));
    System.out.println("training performance f1");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test performance f1");
    System.out.println(new MLMeasures(plugInF1, test));
    LogRiskOptimizer accOptimizer = new LogRiskOptimizer(train, accScorer, cmlcrf, 1, false, false, 1, 1);
    accOptimizer.iterate();
    System.out.println("after ML estimation");
    System.out.println("training with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, train));
    System.out.println("training with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, test));
    System.out.println("test with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, test));
    System.out.println(cmlcrf);
    System.out.println(fOptimizer.objectiveDetail());
    while (!fOptimizer.getTerminator().shouldTerminate()) {
        System.out.println("------------");
        fOptimizer.iterate();
        System.out.println(fOptimizer.objectiveDetail());
        System.out.println("training performance acc");
        System.out.println(new MLMeasures(cmlcrf, train));
        System.out.println("test performance acc");
        System.out.println(new MLMeasures(cmlcrf, test));
        System.out.println("training performance f1");
        System.out.println(new MLMeasures(plugInF1, train));
        System.out.println("test performance f1");
        System.out.println(new MLMeasures(plugInF1, test));
        System.out.println(cmlcrf);
    }
}

Example 24 with MLMeasures

use of edu.neu.ccs.pyramid.eval.MLMeasures in project pyramid by cheng-li.

the class LogRiskOptimizerTest method test4.

private static void test4() {
    MultiLabelClfDataSet train = MultiLabelSynthesizer.crfArgmaxHide();
    MultiLabelClfDataSet test = MultiLabelSynthesizer.crfArgmaxHide();
    CMLCRF cmlcrf = new CMLCRF(train);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(0, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(0).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(1).set(1, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(0, 10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(2).set(1, 0);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(0, -10);
    cmlcrf.getWeights().getWeightsWithoutBiasForClass(3).set(1, -10);
    MLScorer fScorer = new FScorer();
    MLScorer accScorer = new AccScorer();
    SubsetAccPredictor plugInAcc = new SubsetAccPredictor(cmlcrf);
    LogRiskOptimizer fOptimizer = new LogRiskOptimizer(train, fScorer, cmlcrf, 1, false, false, 1, 1);
    InstanceF1Predictor plugInF1 = new InstanceF1Predictor(cmlcrf);
    System.out.println(cmlcrf);
    System.out.println("training performance acc");
    System.out.println(new MLMeasures(cmlcrf, train));
    System.out.println("test performance acc");
    System.out.println(new MLMeasures(cmlcrf, test));
    System.out.println("training performance f1");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test performance f1");
    System.out.println(new MLMeasures(plugInF1, test));
    System.out.println(fOptimizer.objectiveDetail());
    LogRiskOptimizer accOptimizer = new LogRiskOptimizer(train, accScorer, cmlcrf, 1, false, false, 1, 1);
    accOptimizer.iterate();
    System.out.println("after ML estimation");
    System.out.println("training with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, train));
    System.out.println("training with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, train));
    System.out.println("test with Acc predictor");
    System.out.println(new MLMeasures(plugInAcc, test));
    System.out.println("test with F1 predictor");
    System.out.println(new MLMeasures(plugInF1, test));
    System.out.println(cmlcrf);
    System.out.println(fOptimizer.objectiveDetail());
    while (!fOptimizer.getTerminator().shouldTerminate()) {
        System.out.println("------------");
        fOptimizer.iterate();
        System.out.println(fOptimizer.objectiveDetail());
        System.out.println("training performance acc");
        System.out.println(new MLMeasures(cmlcrf, train));
        System.out.println("test performance acc");
        System.out.println(new MLMeasures(cmlcrf, test));
        System.out.println("training performance f1");
        System.out.println(new MLMeasures(plugInF1, train));
        System.out.println("test performance f1");
        System.out.println(new MLMeasures(plugInF1, test));
        System.out.println(cmlcrf);
    }
}