Examples with EvalConfig ml.shifu.shifu.container.obj.EvalConfig used on opensource projects

Example 6 with EvalConfig

use of ml.shifu.shifu.container.obj.EvalConfig in project shifu by ShifuML.

the class PerformanceEvaluatorTest method reviewTest.

@Test(expectedExceptions = FileNotFoundException.class)
public void reviewTest() throws IOException {
    ModelConfig model = ModelConfig.createInitModelConfig("test", ALGORITHM.NN, ".", false);
    EvalConfig eval = new EvalConfig();
    eval.setName("test");
    eval.setDataSet(new RawSourceData());
    model.getBasic().setRunMode(RunMode.LOCAL);
    PerformanceEvaluator actor = new PerformanceEvaluator(model, eval);
    actor.review();
}

Example 7 with EvalConfig

use of ml.shifu.shifu.container.obj.EvalConfig in project shifu by ShifuML.

the class DataPurifierTest method testIsFilterOutA.

@Test
public void testIsFilterOutA() throws IOException {
    dataPurifier = new DataPurifier(modelConfig, false);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    modelConfig.getDataSet().setFilterExpressions("aaa == aaa");
    dataPurifier = new DataPurifier(modelConfig, false);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    modelConfig.getDataSet().setFilterExpressions("1 == 2");
    dataPurifier = new DataPurifier(modelConfig, false);
    Assert.assertFalse(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    modelConfig.getDataSet().setFilterExpressions("*");
    dataPurifier = new DataPurifier(modelConfig, false);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    EvalConfig evalConfig = modelConfig.getEvalConfigByName("EvalA");
    evalConfig.getDataSet().setFilterExpressions("diagnosis == \"M\"");
    dataPurifier = new DataPurifier(evalConfig);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    Assert.assertFalse(dataPurifier.isFilter("B|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    evalConfig.getDataSet().setFilterExpressions("  ");
    dataPurifier = new DataPurifier(evalConfig);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    Assert.assertTrue(dataPurifier.isFilter("B|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    evalConfig.getDataSet().setFilterExpressions(" ASDF *** SDFKSADFJKS >  ");
    dataPurifier = new DataPurifier(evalConfig);
    Assert.assertTrue(dataPurifier.isFilter("M|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
    Assert.assertTrue(dataPurifier.isFilter("B|17.99|10.38|122.8|1001|0.1184|0.2776|0.3001|0.1471|0.2419|0.07871|1.095|0.9053|8.589|153.4|0.006399|0.04904|0.05373|0.01587|0.03003|0.006193|25.38|17.33|184.6|2019|0.1622|0.6656|0.7119|0.2654|0.4601|0.1189"));
}

Example 8 with EvalConfig

use of ml.shifu.shifu.container.obj.EvalConfig in project shifu by ShifuML.

the class ModelSpecLoaderUtilsTest method testFindModels.

@Test
public void testFindModels() throws IOException {
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", SourceType.LOCAL);
    File srcModels = new File("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models");
    File dstModels = new File("models");
    FileUtils.copyDirectory(srcModels, dstModels);
    List<FileStatus> modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, null, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    EvalConfig evalConfig = modelConfig.getEvalConfigByName("EvalA");
    evalConfig.setCustomPaths(new HashMap<String, String>());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, null);
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "  ");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    FileUtils.deleteDirectory(dstModels);
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models/model0.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(1, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "not-exists");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(0, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models/*.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet{0,1,9}/*/*.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
}

Example 9 with EvalConfig

use of ml.shifu.shifu.container.obj.EvalConfig in project shifu by ShifuML.

the class ModelDataEncodeProcessor method run.

public int run() {
    LOG.info("Step Start: encode");
    int status = 0;
    try {
        setUp(ModelStep.ENCODE);
        syncDataToHdfs(modelConfig.getDataSet().getSource());
        String encodeRefModel = getEncodeRefModel();
        if (StringUtils.isNotBlank(encodeRefModel) && !ShifuFileUtils.isFileExists(encodeRefModel, RawSourceData.SourceType.LOCAL)) {
            // user set the encode reference model, but it doesn't exist yet
            File tmpDir = Files.createTempDir();
            ShifuFileUtils.copy(".", tmpDir.getPath(), RawSourceData.SourceType.LOCAL);
            ShifuFileUtils.copy(tmpDir.getPath(), encodeRefModel, RawSourceData.SourceType.LOCAL);
            FileUtils.deleteDirectory(tmpDir);
            updateModel(encodeRefModel);
        }
        String encodeDataSet = getEncodeDataSet();
        if (TRAINING_DATA_SET.equals(encodeDataSet)) {
            // run data encode for training data set
            status = encodeModelData(null);
            updateTrainEncodeDataPath(status, encodeRefModel);
        } else if (StringUtils.isNotBlank(encodeDataSet)) {
            // run data encode for evaluation data set
            List<EvalConfig> evalConfigList = getEvalConfigList(encodeDataSet);
            if (CollectionUtils.isEmpty(evalConfigList)) {
                LOG.error("The EvalSet(s) specified - {} doesn't exits. Please check!", encodeDataSet);
                status = 1;
            } else {
                for (EvalConfig evalConfig : evalConfigList) {
                    status = encodeModelData(evalConfig);
                    updateEvalEncodeDataPath(status, encodeRefModel, evalConfig);
                    if (status != 0) {
                        break;
                    }
                }
            }
        } else {
            // run data encode for all data set
            // run data encode for training data set
            status = encodeModelData(null);
            updateTrainEncodeDataPath(status, encodeRefModel);
            // run data encode for all evaluation data set
            List<EvalConfig> evalConfigList = this.modelConfig.getEvals();
            if (status == 0 && CollectionUtils.isNotEmpty(evalConfigList)) {
                for (EvalConfig evalConfig : evalConfigList) {
                    status = encodeModelData(evalConfig);
                    updateEvalEncodeDataPath(status, encodeRefModel, evalConfig);
                    if (status != 0) {
                        break;
                    }
                }
            }
        }
        clearUp(ModelStep.ENCODE);
    } catch (Exception e) {
        LOG.error("Fail to run encoding.", e);
        status = 1;
    }
    return status;
}

Example 10 with EvalConfig

use of ml.shifu.shifu.container.obj.EvalConfig in project shifu by ShifuML.

the class BasicModelProcessor method syncDataToHdfs.

/**
 * Sync data into HDFS for list of EvalConfig,
 *
 * @param evalConfigList
 *            - EvalConfig list to sync
 * @return true if synced to HDFS
 * @throws IOException
 *             if exception in creating eval folder
 */
protected boolean syncDataToHdfs(List<EvalConfig> evalConfigList) throws IOException {
    if (CollectionUtils.isNotEmpty(evalConfigList)) {
        // create local eval folder
        for (EvalConfig evalConfig : evalConfigList) {
            String evalSetPath = pathFinder.getEvalSetPath(evalConfig, SourceType.LOCAL);
            FileUtils.forceMkdir(new File(evalSetPath));
        }
        // sync files to HDFS
        for (EvalConfig evalConfig : evalConfigList) {
            if (SourceType.HDFS.equals(evalConfig.getDataSet().getSource())) {
                return syncDataToHdfs(evalConfig.getDataSet().getSource());
            }
        }
    }
    return false;
}