Examples with WorkUnitState org.apache.gobblin.configuration.WorkUnitState used on opensource projects

Example 96 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class JobState method getTaskStatesAsWorkUnitStates.

/**
 * Get task states of {@link Task}s of this job as {@link WorkUnitState}s.
 *
 * @return a list of {@link WorkUnitState}s
 */
public List<WorkUnitState> getTaskStatesAsWorkUnitStates() {
    ImmutableList.Builder<WorkUnitState> builder = ImmutableList.builder();
    for (TaskState taskState : this.taskStates.values()) {
        WorkUnitState workUnitState = new WorkUnitState(taskState.getWorkunit(), taskState.getJobState());
        workUnitState.setId(taskState.getId());
        workUnitState.addAll(taskState);
        builder.add(workUnitState);
    }
    return builder.build();
}

Example 97 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class TaskContinuousTest method getStreamingTaskState.

private TaskState getStreamingTaskState() {
    WorkUnitState workUnitState = new WorkUnitState(WorkUnit.create(new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName())));
    workUnitState.setProp(ConfigurationKeys.TASK_KEY_KEY, "1234");
    TaskState taskState = new TaskState(workUnitState);
    taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false));
    taskState.setProp(TaskConfigurationKeys.TASK_EXECUTION_MODE, ExecutionModel.STREAMING.name());
    taskState.setJobId("1234");
    taskState.setTaskId("testContinuousTaskId");
    return taskState;
}

Example 98 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class TaskStateTest method setUp.

@BeforeClass
public void setUp() {
    WorkUnitState workUnitState = new WorkUnitState();
    workUnitState.setProp(ConfigurationKeys.JOB_ID_KEY, "Job-1");
    workUnitState.setProp(ConfigurationKeys.TASK_ID_KEY, "Task-1");
    this.taskState = new TaskState(workUnitState);
}

Example 99 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class MultiConverterTest method testConversionWithMultiplicityAndOneOrEmpty.

/**
 * Combines {@link MultiIdentityConverter()} with {@link OneOrEmptyConverter()}
 * @throws Exception
 */
@Test
public void testConversionWithMultiplicityAndOneOrEmpty() throws Exception {
    MultiConverter multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new MultiIdentityConverter(20), new OneOrEmptyConverter(1), new TestConverter()));
    WorkUnitState workUnitState = new WorkUnitState();
    Schema schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Iterable<Object> convertedRecordIterable = multiConverter.convertRecord(schema, TEST_RECORD, workUnitState);
    Assert.assertEquals(Iterables.size(convertedRecordIterable), 20);
    for (Object record : convertedRecordIterable) {
        checkConvertedAvroData(schema, (GenericRecord) record);
    }
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new MultiIdentityConverter(20), new OneOrEmptyConverter(10), new TestConverter()));
    workUnitState = new WorkUnitState();
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    convertedRecordIterable = multiConverter.convertRecord(schema, TEST_RECORD, workUnitState);
    Assert.assertEquals(Iterables.size(convertedRecordIterable), 2);
    for (Object record : convertedRecordIterable) {
        checkConvertedAvroData(schema, (GenericRecord) record);
    }
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new MultiIdentityConverter(20), new OneOrEmptyConverter(10), new MultiIdentityConverter(10), new TestConverter()));
    workUnitState = new WorkUnitState();
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    convertedRecordIterable = multiConverter.convertRecord(schema, TEST_RECORD, workUnitState);
    Assert.assertEquals(Iterables.size(convertedRecordIterable), 20);
    for (Object record : convertedRecordIterable) {
        checkConvertedAvroData(schema, (GenericRecord) record);
    }
}

Example 100 with WorkUnitState

use of org.apache.gobblin.configuration.WorkUnitState in project incubator-gobblin by apache.

the class MultiConverterTest method testConversionWithEmptyConverter.

@Test
public void testConversionWithEmptyConverter() throws Exception {
    WorkUnitState workUnitState = new WorkUnitState();
    MultiConverter multiConverter = new MultiConverter(Lists.newArrayList(new EmptyConverter(), new SchemaSimplificationConverter(), new TestConverter()));
    Schema schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Assert.assertFalse(multiConverter.convertRecord(schema, TEST_RECORD, workUnitState).iterator().hasNext());
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new EmptyConverter(), new TestConverter()));
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Assert.assertFalse(multiConverter.convertRecord(schema, TEST_RECORD, workUnitState).iterator().hasNext());
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new TestConverter(), new EmptyConverter()));
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Assert.assertFalse(multiConverter.convertRecord(schema, TEST_RECORD, workUnitState).iterator().hasNext());
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new MultiIdentityConverter(5), new TestConverter(), new EmptyConverter()));
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Assert.assertFalse(multiConverter.convertRecord(schema, TEST_RECORD, workUnitState).iterator().hasNext());
    multiConverter = new MultiConverter(Lists.newArrayList(new SchemaSimplificationConverter(), new EmptyConverter(), new MultiIdentityConverter(5), new TestConverter()));
    schema = (Schema) multiConverter.convertSchema(TEST_SCHEMA, workUnitState);
    Assert.assertFalse(multiConverter.convertRecord(schema, TEST_RECORD, workUnitState).iterator().hasNext());
}