Examples with RowLevelPolicyCheckerBuilderFactory org.apache.gobblin.qualitychecker.row.RowLevelPolicyCheckerBuilderFactory used on opensource projects

Example 1 with RowLevelPolicyCheckerBuilderFactory

use of org.apache.gobblin.qualitychecker.row.RowLevelPolicyCheckerBuilderFactory in project incubator-gobblin by apache.

the class RowLevelQualityCheckerTest method testRowLevelPolicy.

@Test(groups = { "ignore" })
public void testRowLevelPolicy() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.ROW_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestRowLevelPolicy");
    state.setProp(ConfigurationKeys.ROW_LEVEL_POLICY_LIST_TYPE, "FAIL");
    RowLevelPolicyChecker checker = new RowLevelPolicyCheckerBuilderFactory().newPolicyCheckerBuilder(state, -1).build();
    RowLevelPolicyCheckResults results = new RowLevelPolicyCheckResults();
    FileReader<GenericRecord> fileReader = openFile(state);
    for (GenericRecord datum : fileReader) {
        Assert.assertTrue(checker.executePolicies(datum, results));
    }
}

Example 2 with RowLevelPolicyCheckerBuilderFactory

use of org.apache.gobblin.qualitychecker.row.RowLevelPolicyCheckerBuilderFactory in project incubator-gobblin by apache.

the class RowLevelQualityCheckerTest method testWriteToErrFile.

@Test(groups = { "ignore" })
public void testWriteToErrFile() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.ROW_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestRowLevelPolicyFail");
    state.setProp(ConfigurationKeys.ROW_LEVEL_POLICY_LIST_TYPE, "ERR_FILE");
    state.setProp(ConfigurationKeys.ROW_LEVEL_ERR_FILE, TestConstants.TEST_ERR_FILE);
    state.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI);
    RowLevelPolicyChecker checker = new RowLevelPolicyCheckerBuilderFactory().newPolicyCheckerBuilder(state, -1).build();
    RowLevelPolicyCheckResults results = new RowLevelPolicyCheckResults();
    FileReader<GenericRecord> fileReader = openFile(state);
    for (GenericRecord datum : fileReader) {
        Assert.assertFalse(checker.executePolicies(datum, results));
    }
    FileSystem fs = FileSystem.get(new URI(TestConstants.TEST_FS_URI), new Configuration());
    Path outputPath = new Path(TestConstants.TEST_ERR_FILE, state.getProp(ConfigurationKeys.ROW_LEVEL_POLICY_LIST).replaceAll("\\.", "-") + ".err");
    Assert.assertTrue(fs.exists(outputPath));
    fs.delete(new Path(TestConstants.TEST_ERR_FILE), true);
}