Examples with ModelConfig ml.shifu.shifu.container.obj.ModelConfig used on opensource projects

Example 1 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class PurifyDataUDFTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    instanceC = new PurifyDataUDF("LOCAL", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json", "EvalA");
    instanceB = new PurifyDataUDF("LOCAL", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json");
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", SourceType.LOCAL);
    modelConfig.getDataSet().setFilterExpressions("diagnosis == \"B\"");
    modelConfig.getEvalConfigByName("EvalA").getDataSet().setFilterExpressions("column_11");
    Writer writer = ShifuFileUtils.getWriter("ModelConfig.json", SourceType.LOCAL);
    JSONUtils.writeValue(writer, modelConfig);
    writer.close();
    instanceA = new PurifyDataUDF("LOCAL", "ModelConfig.json", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json");
    instanceD = new PurifyDataUDF("LOCAL", "ModelConfig.json", "src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json", "EvalA");
}

Example 2 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class CommonUtilsTest method loadModelConfigTest.

@Test
public void loadModelConfigTest() throws JsonGenerationException, JsonMappingException, IOException {
    ModelConfig config = ModelConfig.createInitModelConfig(".", ALGORITHM.NN, "test", false);
    config.setModelSetName("shifu");
    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(new File("ModelConfig.json"), config);
    ModelConfig anotherConfig = CommonUtils.loadModelConfig();
    Assert.assertEquals(config, anotherConfig);
    FileUtils.deleteQuietly(new File("ModelConfig.json"));
}

Example 3 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class CommonUtilsTest method testLoadModelConfig.

@Test
public void testLoadModelConfig() throws IOException {
    ModelConfig config = CommonUtils.loadModelConfig("src/test/resources/example/wdbc/wdbcModelSetLocal/ModelConfig.json", SourceType.LOCAL);
    Assert.assertEquals(config.getDataSet().getNegTags().get(0), "B");
}

Example 4 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class BaggingSubsampleUDFTest method setUp.

@BeforeClass
public void setUp() throws Exception {
    File file = new File("udf");
    if (!file.exists()) {
        FileUtils.forceMkdir(file);
    }
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", SourceType.LOCAL);
    List<ColumnConfig> columnConfigList = CommonUtils.loadColumnConfigList("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ColumnConfig.json", SourceType.LOCAL);
    modelConfig.getTrain().setBaggingNum(1);
    ;
    modelConfig.getTrain().setBaggingSampleRate(2.0);
    ;
    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(new File("udf/ModelConfig.json"), modelConfig);
    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(new File("udf/ColumnConfig.json"), columnConfigList);
}

Example 5 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelDataEncodeProcessor method updateEvalEncodeDataPath.

private void updateEvalEncodeDataPath(int status, String encodeRefModel, EvalConfig evalConfig) throws IOException {
    if (status == 0 && StringUtils.isNotBlank(encodeRefModel)) {
        String delimiter = Environment.getProperty(Constants.SHIFU_OUTPUT_DATA_DELIMITER, Constants.DEFAULT_DELIMITER);
        String encodeDataPath = this.pathFinder.getEncodeDataPath(evalConfig);
        ModelConfig encodeModel = loadSubModelConfig(encodeRefModel);
        EvalConfig encodeEvalConfig = encodeModel.getEvalConfigByName(evalConfig.getName());
        if (encodeEvalConfig == null) {
            // new EvalSet, add it to encode model
            encodeEvalConfig = evalConfig.clone();
            encodeModel.getEvals().add(encodeEvalConfig);
        }
        encodeEvalConfig.getDataSet().setDataPath(encodeDataPath);
        encodeEvalConfig.getDataSet().setDataDelimiter(delimiter);
        encodeEvalConfig.getDataSet().setHeaderPath(encodeDataPath + File.separator + ".pig_header");
        encodeEvalConfig.getDataSet().setHeaderDelimiter(delimiter);
        // remove filter
        encodeEvalConfig.getDataSet().setFilterExpressions("");
        saveModelConfig(encodeRefModel, encodeModel);
    }
}