Examples with TaskLevelPolicyCheckResults org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults used on opensource projects

Example 1 with TaskLevelPolicyCheckResults

use of org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults in project incubator-gobblin by apache.

the class Fork method checkDataQuality.

/**
 * Check data quality.
 *
 * @return whether data publishing is successful and data should be committed
 */
private boolean checkDataQuality(Optional<Object> schema) throws Exception {
    if (this.branches > 1) {
        this.forkTaskState.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, this.taskState.getProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED));
        this.forkTaskState.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED, this.taskState.getProp(ConfigurationKeys.EXTRACTOR_ROWS_EXTRACTED));
    }
    String writerRecordsWrittenKey = ForkOperatorUtils.getPropertyNameForBranch(ConfigurationKeys.WRITER_RECORDS_WRITTEN, this.branches, this.index);
    if (this.writer.isPresent()) {
        this.forkTaskState.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, this.writer.get().recordsWritten());
        this.taskState.setProp(writerRecordsWrittenKey, this.writer.get().recordsWritten());
    } else {
        this.forkTaskState.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, 0L);
        this.taskState.setProp(writerRecordsWrittenKey, 0L);
    }
    if (schema.isPresent()) {
        this.forkTaskState.setProp(ConfigurationKeys.EXTRACT_SCHEMA, schema.get().toString());
    }
    try {
        // Do task-level quality checking
        TaskLevelPolicyCheckResults taskResults = this.taskContext.getTaskLevelPolicyChecker(this.forkTaskState, this.branches > 1 ? this.index : -1).executePolicies();
        TaskPublisher publisher = this.taskContext.getTaskPublisher(this.forkTaskState, taskResults);
        switch(publisher.canPublish()) {
            case SUCCESS:
                return true;
            case CLEANUP_FAIL:
                this.logger.error("Cleanup failed for task " + this.taskId);
                break;
            case POLICY_TESTS_FAIL:
                this.logger.error("Not all quality checking passed for task " + this.taskId);
                break;
            case COMPONENTS_NOT_FINISHED:
                this.logger.error("Not all components completed for task " + this.taskId);
                break;
            default:
                break;
        }
        return false;
    } catch (Throwable t) {
        this.logger.error("Failed to check task-level data quality", t);
        return false;
    }
}

Example 2 with TaskLevelPolicyCheckResults

use of org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults in project incubator-gobblin by apache.

the class RowCountTaskLevelPolicyTest method testMultipleRowCountPolicies.

@Test
public void testMultipleRowCountPolicies() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.count.RowCountRangePolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL");
    state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, EXTRACTOR_ROWS_READ);
    state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, WRITER_ROWS_WRITTEN);
    state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, "0.05");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
    }
}

Example 3 with TaskLevelPolicyCheckResults

use of org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults in project incubator-gobblin by apache.

the class RowCountTaskLevelPolicyTest method testRowCountRangePolicyFailed.

@Test
public void testRowCountRangePolicyFailed() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.policies.count.RowCountRangePolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL");
    state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, EXTRACTOR_ROWS_READ);
    state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, -1);
    state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, "0.05");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.FAILED);
    }
}

Example 4 with TaskLevelPolicyCheckResults

use of org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults in project incubator-gobblin by apache.

the class TaskLevelQualityCheckerTest method testMultiplePolicies.

@Test
public void testMultiplePolicies() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy,org.apache.gobblin.qualitychecker.TestTaskLevelPolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
    }
}

Example 5 with TaskLevelPolicyCheckResults

use of org.apache.gobblin.qualitychecker.task.TaskLevelPolicyCheckResults in project incubator-gobblin by apache.

the class RowCountTaskLevelPolicyTest method testRowCountPolicyPassed.

@Test
public void testRowCountPolicyPassed() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.policies.count.RowCountPolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL");
    state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, EXTRACTOR_ROWS_READ);
    state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, WRITER_ROWS_WRITTEN);
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
    }
}