Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 56 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class BaseDataPublisherTest method testNoOutputWhenDisabled.

@Test
public void testNoOutputWhenDisabled() throws IOException {
    State s = buildDefaultState(1);
    WorkUnitState wuState = new WorkUnitState();
    addStateToWorkunit(s, wuState);
    wuState.setProp(ConfigurationKeys.WRITER_METADATA_KEY, "abcdefg");
    BaseDataPublisher publisher = new BaseDataPublisher(s);
    publisher.publishMetadata(Collections.singletonList(wuState));
    File mdFile = openMetadataFile(s, 1, 0);
    Assert.assertFalse(mdFile.exists(), "Internal metadata from writer should not be written out if no merger is set in config");
}

Example 57 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RowCountTaskLevelPolicyTest method testMultipleRowCountPolicies.

@Test
public void testMultipleRowCountPolicies() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.policies.count.RowCountPolicy,org.apache.gobblin.policies.count.RowCountRangePolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL");
    state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, EXTRACTOR_ROWS_READ);
    state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, WRITER_ROWS_WRITTEN);
    state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, "0.05");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
    }
}

Example 58 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RowCountTaskLevelPolicyTest method testRowCountRangePolicyFailed.

@Test
public void testRowCountRangePolicyFailed() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.policies.count.RowCountRangePolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL");
    state.setProp(ConfigurationKeys.EXTRACTOR_ROWS_EXPECTED, EXTRACTOR_ROWS_READ);
    state.setProp(ConfigurationKeys.WRITER_ROWS_WRITTEN, -1);
    state.setProp(ConfigurationKeys.ROW_COUNT_RANGE, "0.05");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.FAILED);
    }
}

Example 59 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class TaskLevelQualityCheckerTest method testMultiplePolicies.

@Test
public void testMultiplePolicies() throws Exception {
    State state = new State();
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST, "org.apache.gobblin.qualitychecker.TestTaskLevelPolicy,org.apache.gobblin.qualitychecker.TestTaskLevelPolicy");
    state.setProp(ConfigurationKeys.TASK_LEVEL_POLICY_LIST_TYPE, "FAIL,FAIL");
    TaskLevelPolicyCheckResults results = getPolicyResults(state);
    for (Map.Entry<TaskLevelPolicy.Result, TaskLevelPolicy.Type> entry : results.getPolicyResults().entrySet()) {
        Assert.assertEquals(entry.getKey(), TaskLevelPolicy.Result.PASSED);
    }
}

Example 60 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RetryWriter method getFinalState.

@Override
public State getFinalState() {
    State state = new State();
    if (this.writer instanceof FinalState) {
        state.addAll(((FinalState) this.writer).getFinalState());
    } else {
        LOG.warn("Wrapped writer does not implement FinalState: " + this.writer.getClass());
    }
    state.setProp(FAILED_WRITES_KEY, this.failedWrites);
    return state;
}