Examples with ModelConfig ml.shifu.shifu.container.obj.ModelConfig used on opensource projects

Example 26 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class CommonUtilsTest method syncUpEvalTest.

// @Test
public void syncUpEvalTest() throws IOException {
    ModelConfig config = ModelConfig.createInitModelConfig(".", ALGORITHM.NN, "test", false);
    config.setModelSetName("shifu");
    File file = new File("evals/EvalA");
    if (!file.exists()) {
        FileUtils.forceMkdir(file);
    }
    file = new File("testEval/EvalConfig.json");
    FileUtils.touch(file);
    // CommonUtils.copyEvalConfFromLocalToHDFS(config, "testEval");
    Assert.assertTrue(file.exists());
    FileUtils.deleteDirectory(new File("ModelSets"));
    FileUtils.deleteDirectory(new File("evals"));
}

Example 27 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelSpecLoaderUtilsTest method testFindModels.

@Test
public void testFindModels() throws IOException {
    ModelConfig modelConfig = CommonUtils.loadModelConfig("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/ModelConfig.json", SourceType.LOCAL);
    File srcModels = new File("src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models");
    File dstModels = new File("models");
    FileUtils.copyDirectory(srcModels, dstModels);
    List<FileStatus> modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, null, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    EvalConfig evalConfig = modelConfig.getEvalConfigByName("EvalA");
    evalConfig.setCustomPaths(new HashMap<String, String>());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, null);
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "  ");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    FileUtils.deleteDirectory(dstModels);
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models/model0.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(1, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "not-exists");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(0, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet1/models/*.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
    evalConfig.getCustomPaths().put(Constants.KEY_MODELS_PATH, "./src/test/resources/example/cancer-judgement/ModelStore/ModelSet{0,1,9}/*/*.nn");
    modelFiles = ModelSpecLoaderUtils.findModels(modelConfig, evalConfig, SourceType.LOCAL);
    Assert.assertEquals(5, modelFiles.size());
}

Example 28 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class VarSelMaster method init.

private void init(MasterContext<VarSelMasterResult, VarSelWorkerResult> context) {
    Properties props = context.getProps();
    try {
        SourceType sourceType = SourceType.valueOf(props.getProperty(CommonConstants.MODELSET_SOURCE_TYPE, SourceType.HDFS.toString()));
        this.modelConfig = CommonUtils.loadModelConfig(props.getProperty(CommonConstants.SHIFU_MODEL_CONFIG), sourceType);
        this.columnConfigList = CommonUtils.loadColumnConfigList(props.getProperty(CommonConstants.SHIFU_COLUMN_CONFIG), sourceType);
        String conductorClsName = props.getProperty(Constants.VAR_SEL_MASTER_CONDUCTOR);
        this.masterConductor = (AbstractMasterConductor) Class.forName(conductorClsName).getDeclaredConstructor(ModelConfig.class, List.class).newInstance(this.modelConfig, this.columnConfigList);
    } catch (IOException e) {
        throw new RuntimeException("Fail to load ModelConfig or List<ColumnConfig>", e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException("Invalid Master Conductor class", e);
    } catch (InstantiationException e) {
        e.printStackTrace();
    } catch (IllegalAccessException e) {
        e.printStackTrace();
    } catch (NoSuchMethodException e) {
        e.printStackTrace();
    } catch (InvocationTargetException e) {
        e.printStackTrace();
    }
}

Example 29 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class VarSelWorker method init.

@Override
public void init(WorkerContext<VarSelMasterResult, VarSelWorkerResult> workerContext) {
    Properties props = workerContext.getProps();
    try {
        RawSourceData.SourceType sourceType = RawSourceData.SourceType.valueOf(props.getProperty(CommonConstants.MODELSET_SOURCE_TYPE, RawSourceData.SourceType.HDFS.toString()));
        this.modelConfig = CommonUtils.loadModelConfig(props.getProperty(CommonConstants.SHIFU_MODEL_CONFIG), sourceType);
        this.columnConfigList = CommonUtils.loadColumnConfigList(props.getProperty(CommonConstants.SHIFU_COLUMN_CONFIG), sourceType);
        String conductorClsName = props.getProperty(Constants.VAR_SEL_WORKER_CONDUCTOR);
        this.workerConductor = (AbstractWorkerConductor) Class.forName(conductorClsName).getDeclaredConstructor(ModelConfig.class, List.class).newInstance(this.modelConfig, this.columnConfigList);
    } catch (IOException e) {
        throw new RuntimeException("Fail to load ModelConfig or List<ColumnConfig>", e);
    } catch (ClassNotFoundException e) {
        throw new RuntimeException("Invalid Master Conductor class", e);
    } catch (InstantiationException e) {
        throw new RuntimeException("Fail to create instance", e);
    } catch (IllegalAccessException e) {
        throw new RuntimeException("Illegal access when creating instance", e);
    } catch (NoSuchMethodException e) {
        throw new RuntimeException("Fail to call method when creating instance", e);
    } catch (InvocationTargetException e) {
        throw new RuntimeException("Fail to invoke when creating instance", e);
    }
    List<Integer> normalizedColumnIdList = this.getNormalizedColumnIdList();
    this.inputNodeCount = normalizedColumnIdList.size();
    this.outputNodeCount = this.getTargetColumnCount();
    trainingDataSet = new TrainingDataSet(normalizedColumnIdList);
    try {
        dataPurifier = new DataPurifier(modelConfig, false);
    } catch (IOException e) {
        throw new RuntimeException("Fail to create DataPurifier", e);
    }
    this.targetColumnId = CommonUtils.getTargetColumnNum(this.columnConfigList);
    if (StringUtils.isNotBlank(modelConfig.getWeightColumnName())) {
        for (ColumnConfig columnConfig : columnConfigList) {
            if (columnConfig.getColumnName().equalsIgnoreCase(modelConfig.getWeightColumnName().trim())) {
                this.weightColumnId = columnConfig.getColumnNum();
                break;
            }
        }
    }
}

Example 30 with ModelConfig

use of ml.shifu.shifu.container.obj.ModelConfig in project shifu by ShifuML.

the class ModelDataEncodeProcessor method updateTrainEncodeDataPath.

private void updateTrainEncodeDataPath(int status, String encodeRefModel) throws IOException {
    if (status == 0 && StringUtils.isNotBlank(encodeRefModel)) {
        String delimiter = Environment.getProperty(Constants.SHIFU_OUTPUT_DATA_DELIMITER, Constants.DEFAULT_DELIMITER);
        String encodeDataPath = this.pathFinder.getEncodeDataPath(null);
        ModelConfig encodeModel = loadSubModelConfig(encodeRefModel);
        encodeModel.getDataSet().setDataPath(encodeDataPath);
        encodeModel.getDataSet().setDataDelimiter(delimiter);
        encodeModel.getDataSet().setHeaderPath(encodeDataPath + File.separator + ".pig_header");
        encodeModel.getDataSet().setHeaderDelimiter(delimiter);
        // remove filter
        encodeModel.getDataSet().setFilterExpressions("");
        saveModelConfig(encodeRefModel, encodeModel);
    }
}