Examples with ModelConfig ml.shifu.shifu.container.obj.ModelConfig used on opensource projects

Example 21 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelInspectorTest method testValidateMeta.

@Test
public void testValidateMeta() throws Exception {
    ModelConfig config = CommonUtils.loadModelConfig();
    instance.checkMeta(config);
}

Example 22 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelInspectorTest method testValidateNullStep.

@Test
public void testValidateNullStep() throws Exception {
    ModelConfig config = CommonUtils.loadModelConfig();
    ValidateResult result = instance.probe(config, null);
    Assert.assertTrue(result.getStatus());
}

Example 23 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class PigExecutorTest method test.

// @Test
public void test() throws IOException {
    PigExecutor exec = PigExecutor.getExecutor();
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", SourceType.LOCAL);
    Environment.setProperty(Environment.SHIFU_HOME, ".");
    modelConfig.getDataSet().setSource(SourceType.LOCAL);
    exec.submitJob(modelConfig, "src/test/java/ml/shifu/shifu/pig/pigTest.pig");
    FileUtils.deleteQuietly(new File("ModelConfig.json"));
    FileUtils.deleteDirectory(new File("ModelSets"));
}

Example 24 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class CommonUtilsTest method syncTest.

// @Test
public void syncTest() throws IOException {
    ModelConfig config = ModelConfig.createInitModelConfig(".", ALGORITHM.NN, "test", false);
    config.setModelSetName("testModel");
    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(new File("ModelConfig.json"), config);
    ColumnConfig col = new ColumnConfig();
    col.setColumnName("ColumnA");
    List<ColumnConfig> columnConfigList = new ArrayList<ColumnConfig>();
    columnConfigList.add(col);
    config.getDataSet().setSource(SourceType.LOCAL);
    ;
    jsonMapper.writerWithDefaultPrettyPrinter().writeValue(new File("ColumnConfig.json"), columnConfigList);
    File file = null;
    file = new File("models");
    if (!file.exists()) {
        FileUtils.forceMkdir(file);
    }
    file = new File("models/model1.nn");
    if (!file.exists()) {
        if (file.createNewFile()) {
            BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), Constants.DEFAULT_CHARSET));
            writer.write("test string");
            writer.close();
        } else {
            LOG.warn("Create file {} failed", file.getAbsolutePath());
        }
    }
    file = new File("EvalSets/test");
    if (!file.exists()) {
        FileUtils.forceMkdir(file);
    }
    file = new File("EvalSets/test/EvalConfig.json");
    if (!file.exists()) {
        BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), Constants.DEFAULT_CHARSET));
        writer.write("test string");
        writer.close();
    }
    CommonUtils.copyConfFromLocalToHDFS(config, new PathFinder(config));
    file = new File("ModelSets");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel/ModelConfig.json");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel/ColumnConfig.json");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel/ReasonCodeMap.json");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel/models/model1.nn");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets/testModel/EvalSets/test/EvalConfig.json");
    Assert.assertTrue(file.exists());
    file = new File("ModelSets");
    if (file.exists()) {
        FileUtils.deleteDirectory(file);
    }
    file = new File("ColumnConfig.json");
    FileUtils.deleteQuietly(file);
    file = new File("ModelConfig.json");
    FileUtils.deleteQuietly(file);
    FileUtils.deleteDirectory(new File("models"));
    FileUtils.deleteDirectory(new File("EvalSets"));
}

Example 25 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class CommonUtilsTest method updateColumnConfigFlagsTest.

// @Test
public void updateColumnConfigFlagsTest() throws IOException {
    ModelConfig config = ModelConfig.createInitModelConfig("test", ALGORITHM.NN, "test", false);
    config.getDataSet().setMetaColumnNameFile("./conf/meta_column_conf.txt");
    config.getVarSelect().setForceRemoveColumnNameFile("./conf/remove_column_list.txt");
    List<ColumnConfig> list = new ArrayList<ColumnConfig>();
    ColumnConfig e = new ColumnConfig();
    e.setColumnName("a");
    list.add(e);
    e = new ColumnConfig();
    e.setColumnName("c");
    list.add(e);
    e = new ColumnConfig();
    e.setColumnName("d");
    list.add(e);
    ColumnConfigUpdater.updateColumnConfigFlags(config, list, ModelInspector.ModelStep.VARSELECT);
    Assert.assertTrue(list.get(0).isMeta());
}