Examples with SourceState org.apache.gobblin.configuration.SourceState used on opensource projects

Example 56 with SourceState

use of org.apache.gobblin.configuration.SourceState in project incubator-gobblin by apache.

the class AbstractSourceTest method testGetPreviousWorkUnitStatesWithConfigOverWrittenDisabled.

/**
 * Test under always-retry policy, the overwrite_configs_in_statestore disabled (default).
 * The previous workUnitState would not be reset with the config in the current source.
 */
@Test
public void testGetPreviousWorkUnitStatesWithConfigOverWrittenDisabled() {
    SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates);
    sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "always");
    // random properties for test
    sourceState.setProp("a", "1");
    sourceState.setProp("b", "2");
    List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState);
    Assert.assertEquals(returnedWorkUnitStates, this.expectedPreviousWorkUnitStates);
    for (WorkUnitState workUnitState : returnedWorkUnitStates) {
        Assert.assertEquals(workUnitState.contains("a"), false);
        Assert.assertEquals(workUnitState.contains("b"), false);
    }
}

Example 57 with SourceState

use of org.apache.gobblin.configuration.SourceState in project incubator-gobblin by apache.

the class AbstractSourceTest method testGetPreviousWorkUnitStatesOnFullRetryFullCommit.

/**
 * Test when work unit retry policy is on full, and the job commit policy is "full".
 */
@Test
public void testGetPreviousWorkUnitStatesOnFullRetryFullCommit() {
    SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates);
    sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "onfull");
    sourceState.setProp(ConfigurationKeys.JOB_COMMIT_POLICY_KEY, "full");
    List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState);
    Assert.assertEquals(returnedWorkUnitStates, this.expectedPreviousWorkUnitStates);
}

Example 58 with SourceState

use of org.apache.gobblin.configuration.SourceState in project incubator-gobblin by apache.

the class AbstractSourceTest method testGetPreviousWorkUnitStatesWithConfigOverWrittenEnabled.

/**
 * Test under always-retry policy, the overwrite_configs_in_statestore enabled.
 * The previous workUnitState should be reset with the config in the current source.
 */
@Test
public void testGetPreviousWorkUnitStatesWithConfigOverWrittenEnabled() {
    for (WorkUnitState workUnitState : this.previousWorkUnitStates) {
        workUnitState.setProp("a", "3");
        workUnitState.setProp("b", "4");
    }
    SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates);
    sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "always");
    sourceState.setProp(ConfigurationKeys.OVERWRITE_CONFIGS_IN_STATESTORE, Boolean.TRUE);
    // random properties for test
    sourceState.setProp("a", "1");
    sourceState.setProp("b", "2");
    List<WorkUnitState> returnedWorkUnitStates = this.testSource.getPreviousWorkUnitStatesForRetry(sourceState);
    for (WorkUnitState workUnitState : returnedWorkUnitStates) {
        Assert.assertEquals(workUnitState.getProp("a"), "1");
        Assert.assertEquals(workUnitState.getProp("b"), "2");
    }
}

Example 59 with SourceState

use of org.apache.gobblin.configuration.SourceState in project incubator-gobblin by apache.

the class AbstractSourceTest method testGetPreviousWorkUnitStatesNeverRetry.

/**
 * Test the never-retry policy.
 */
@Test
public void testGetPreviousWorkUnitStatesNeverRetry() {
    SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates);
    sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_POLICY_KEY, "never");
    Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST);
}

Example 60 with SourceState

use of org.apache.gobblin.configuration.SourceState in project incubator-gobblin by apache.

the class AbstractSourceTest method testGetPreviousWorkUnitStatesDisabledRetry.

/**
 * Test when work unit retry disabled.
 */
@Test
public void testGetPreviousWorkUnitStatesDisabledRetry() {
    SourceState sourceState = new SourceState(new State(), this.previousWorkUnitStates);
    sourceState.setProp(ConfigurationKeys.WORK_UNIT_RETRY_ENABLED_KEY, Boolean.FALSE);
    Assert.assertEquals(this.testSource.getPreviousWorkUnitStatesForRetry(sourceState), Collections.EMPTY_LIST);
}