Examples with ModelSourceDataConf ml.shifu.shifu.container.obj.ModelSourceDataConf used on opensource projects

Example 1 with ModelSourceDataConf

use of ml.shifu.shifu.container.obj.ModelSourceDataConf in project shifu by ShifuML.

the class ShifuTestProcessor method runFilterTest.

private int runFilterTest(ModelConfig modelConfig) throws IOException {
    ModelSourceDataConf dataset = modelConfig.getDataSet();
    if (StringUtils.isBlank(dataset.getFilterExpressions())) {
        LOG.warn("No filter expression set in train dataset. Skip it!");
        return 0;
    }
    LOG.info("Start to test the filter against the training dataset.");
    DataPurifier dataPurifier = new DataPurifier(modelConfig, false);
    int status = doFilterTest(dataPurifier, dataset.getDataPath(), dataset.getSource());
    if (status > 0) {
        return status;
    }
    if (StringUtils.isNotBlank(dataset.getValidationFilterExpressions())) {
        LOG.info("Start to test the filter against the validation dataset.");
        dataPurifier = new DataPurifier(modelConfig, true);
        status = doFilterTest(dataPurifier, dataset.getValidationDataPath(), dataset.getSource());
    }
    return status;
}