Examples with ModelConfig ml.shifu.shifu.container.obj.ModelConfig used on opensource projects

Example 16 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ValidationConductorTest method testRunValidate.

@Test
public void testRunValidate() throws IOException {
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", RawSourceData.SourceType.LOCAL);
    List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json", RawSourceData.SourceType.LOCAL);
    List<Integer> columnIdList = new ArrayList<Integer>();
    boolean hasCandidates = CommonUtils.hasCandidateColumns(columnConfigList);
    for (ColumnConfig columnConfig : columnConfigList) {
        if (columnConfig.isCandidate(hasCandidates)) {
            columnIdList.add(columnConfig.getColumnNum());
        }
    }
    TrainingDataSet trainingDataSet = new TrainingDataSet(columnIdList);
    List<String> recordsList = IOUtils.readLines(new FileInputStream("src/test/resources/example/cancer-judgement/DataStore/DataSet1/part-00"));
    for (String record : recordsList) {
        addRecordIntoTrainDataSet(modelConfig, columnConfigList, trainingDataSet, record);
    }
    Set<Integer> workingList = new HashSet<Integer>();
    for (Integer columnId : trainingDataSet.getDataColumnIdList()) {
        workingList.clear();
        workingList.add(columnId);
        ValidationConductor conductor = new ValidationConductor(modelConfig, columnConfigList, workingList, trainingDataSet);
        double error = conductor.runValidate();
        System.out.println("The error is - " + error + ", for columnId - " + columnId);
    }
}

Example 17 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class WrapperWorkerConductorTest method testWrapperConductor.

@Test
public void testWrapperConductor() throws IOException {
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", RawSourceData.SourceType.LOCAL);
    List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json", RawSourceData.SourceType.LOCAL);
    WrapperWorkerConductor wrapper = new WrapperWorkerConductor(modelConfig, columnConfigList);
    TrainingDataSet trainingDataSet = genTrainingDataSet(modelConfig, columnConfigList);
    wrapper.retainData(trainingDataSet);
    List<Integer> columnIdList = new ArrayList<Integer>();
    for (int i = 2; i < 30; i++) {
        columnIdList.add(i);
    }
    List<CandidateSeed> seedList = new ArrayList<CandidateSeed>();
    for (int i = 0; i < 10; i++) {
        seedList.add(new CandidateSeed(0, columnIdList.subList(i + 1, i + 7)));
    }
    wrapper.consumeMasterResult(new VarSelMasterResult(seedList));
    VarSelWorkerResult workerResult = wrapper.generateVarSelResult();
    Assert.assertNotNull(workerResult);
    Assert.assertTrue(workerResult.getSeedPerfList().size() > 0);
}

Example 18 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelInspectorTest method testValidateVarSelect.

@Test
public void testValidateVarSelect() throws Exception {
    ModelConfig config = CommonUtils.loadModelConfig();
    ValidateResult result = instance.probe(config, ModelStep.VARSELECT);
    Assert.assertTrue(result.getStatus());
    config.getVarSelect().setForceRemoveColumnNameFile("not-exists");
    config.getVarSelect().setForceSelectColumnNameFile("~/not-exists");
    result = instance.probe(config, ModelStep.VARSELECT);
    Assert.assertFalse(result.getStatus());
}

Example 19 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelInspectorTest method testValidateTrain.

@Test
public void testValidateTrain() throws Exception {
    ModelConfig config = CommonUtils.loadModelConfig();
    ValidateResult result = instance.probe(config, ModelStep.TRAIN);
    Assert.assertTrue(result.getStatus());
    config.getTrain().getParams().put(CommonConstants.NUM_HIDDEN_LAYERS, -1);
    result = instance.probe(config, ModelStep.TRAIN);
    Assert.assertFalse(result.getStatus());
    config.getTrain().getParams().put(CommonConstants.NUM_HIDDEN_LAYERS, 2);
    List<Integer> hiddenNodes = new ArrayList<Integer>();
    hiddenNodes.add(10);
    hiddenNodes.add(10);
    config.getTrain().getParams().put(CommonConstants.NUM_HIDDEN_NODES, hiddenNodes);
    List<String> activateFuncs = new ArrayList<String>();
    activateFuncs.add("tanh");
    activateFuncs.add("sigmoid");
    activateFuncs.add("tanh");
    config.getTrain().getParams().put(CommonConstants.ACTIVATION_FUNC, activateFuncs);
    result = instance.probe(config, ModelStep.TRAIN);
    Assert.assertFalse(result.getStatus());
}

Example 20 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelInspectorTest method testValidateInit.

@Test
public void testValidateInit() throws Exception {
    ModelConfig config = CommonUtils.loadModelConfig();
    ValidateResult result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    config.getDataSet().setCategoricalColumnNameFile("not-exists");
    config.getDataSet().setMetaColumnNameFile("~/not-exists");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    config.getDataSet().setMetaColumnNameFile(" ");
    config.getDataSet().setCategoricalColumnNameFile("");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    config.getDataSet().setTargetColumnName(null);
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    config.getDataSet().setTargetColumnName("  ");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    config.getDataSet().setTargetColumnName("a");
    config.getDataSet().setMetaColumnNameFile("meta.names");
    FileUtils.write(new File("meta.names"), "a");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    FileUtils.write(new File("meta.names"), "");
    config.getDataSet().setTargetColumnName("a");
    config.getVarSelect().setForceRemoveColumnNameFile("force.remove.names");
    FileUtils.write(new File("force.remove.names"), "a");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    FileUtils.write(new File("force.remove.names"), "");
    config.getDataSet().setTargetColumnName("a");
    config.getVarSelect().setForceSelectColumnNameFile("force.select.names");
    FileUtils.write(new File("force.select.names"), "a");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    FileUtils.write(new File("force.select.names"), "");
    FileUtils.write(new File("force.remove.names"), "");
    FileUtils.write(new File("meta.names"), "b\nc");
    FileUtils.write(new File("force.select.names"), "c\nd");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    FileUtils.write(new File("force.select.names"), "");
    FileUtils.write(new File("force.remove.names"), "");
    FileUtils.write(new File("meta.names"), "b\nc");
    FileUtils.write(new File("force.remove.names"), "c\nd");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    FileUtils.write(new File("meta.names"), "");
    FileUtils.write(new File("force.remove.names"), "b\nc");
    FileUtils.write(new File("force.select.names"), "c\nd");
    result = instance.probe(config, ModelStep.INIT);
    Assert.assertTrue(result.getStatus());
    Assert.assertEquals("[true]", result.toString());
    FileUtils.deleteQuietly(new File("meta.names"));
    FileUtils.deleteQuietly(new File("force.remove.names"));
    FileUtils.deleteQuietly(new File("force.select.names"));
}