Examples with DataPurifier ml.shifu.shifu.core.DataPurifier used on opensource projects

Example 11 with DataPurifier

use of ml.shifu.shifu.core.DataPurifier in project shifu by ShifuML.

the class ShifuTestProcessor method runFilterTest.

private int runFilterTest(ModelConfig modelConfig) throws IOException {
    ModelSourceDataConf dataset = modelConfig.getDataSet();
    if (StringUtils.isBlank(dataset.getFilterExpressions())) {
        LOG.warn("No filter expression set in train dataset. Skip it!");
        return 0;
    }
    LOG.info("Start to test the filter against the training dataset.");
    DataPurifier dataPurifier = new DataPurifier(modelConfig, false);
    int status = doFilterTest(dataPurifier, dataset.getDataPath(), dataset.getSource());
    if (status > 0) {
        return status;
    }
    if (StringUtils.isNotBlank(dataset.getValidationFilterExpressions())) {
        LOG.info("Start to test the filter against the validation dataset.");
        dataPurifier = new DataPurifier(modelConfig, true);
        status = doFilterTest(dataPurifier, dataset.getValidationDataPath(), dataset.getSource());
    }
    return status;
}