Examples with WorkUnit org.apache.gobblin.source.workunit.WorkUnit used on opensource projects

Example 56 with WorkUnit

use of org.apache.gobblin.source.workunit.WorkUnit in project incubator-gobblin by apache.

the class TestStressTestingSource method testSleepTime.

@Test(enabled = false)
public void testSleepTime() throws DataRecordException, IOException {
    final int MEM_ALLOC_BYTES = 100;
    final int NUM_WORK_UNITS = 1;
    final int SLEEP_TIME_MICRO = 10000;
    final int NUM_RECORDS = 500;
    SourceState state = new SourceState();
    state.setProp(StressTestingSource.NUM_WORK_UNITS_KEY, NUM_WORK_UNITS);
    state.setProp(StressTestingSource.MEM_ALLOC_BYTES_KEY, MEM_ALLOC_BYTES);
    state.setProp(StressTestingSource.SLEEP_TIME_MICRO_KEY, SLEEP_TIME_MICRO);
    state.setProp(StressTestingSource.NUM_RECORDS_KEY, NUM_RECORDS);
    StressTestingSource source = new StressTestingSource();
    List<WorkUnit> wus = source.getWorkunits(state);
    Assert.assertEquals(wus.size(), NUM_WORK_UNITS);
    WorkUnit wu = wus.get(0);
    WorkUnitState wuState = new WorkUnitState(wu, state);
    Extractor<String, byte[]> extractor = source.getExtractor(wuState);
    byte[] record;
    long startTimeNano = System.nanoTime();
    while ((record = extractor.readRecord(null)) != null) {
        Assert.assertEquals(record.length, 100);
    }
    long endTimeNano = System.nanoTime();
    long timeSpentMicro = (endTimeNano - startTimeNano) / (1000);
    // check that there is less than 2 second difference between expected and actual time spent
    Assert.assertTrue(Math.abs(timeSpentMicro - (SLEEP_TIME_MICRO * NUM_RECORDS)) < (2000000), "Time spent " + timeSpentMicro);
}

Example 57 with WorkUnit

use of org.apache.gobblin.source.workunit.WorkUnit in project incubator-gobblin by apache.

the class ParallelRunnerTest method testDeserializeFromFile.

@Test(dependsOnMethods = "testSerializeToFile")
public void testDeserializeFromFile() throws IOException {
    WorkUnit workUnit1 = WorkUnit.createEmpty();
    WorkUnit workUnit2 = WorkUnit.createEmpty();
    try (ParallelRunner parallelRunner = new ParallelRunner(2, this.fs)) {
        parallelRunner.deserializeFromFile(workUnit1, new Path(this.outputPath, "wu1"));
        parallelRunner.deserializeFromFile(workUnit2, new Path(this.outputPath, "wu2"));
    }
    Assert.assertEquals(workUnit1.getPropertyNames().size(), 2);
    Assert.assertEquals(workUnit1.getProp("foo"), "bar");
    Assert.assertEquals(workUnit1.getPropAsInt("a"), 10);
    Assert.assertEquals(workUnit2.getPropertyNames().size(), 2);
    Assert.assertEquals(workUnit2.getProp("foo"), "baz");
    Assert.assertEquals(workUnit2.getPropAsInt("b"), 20);
}

Example 58 with WorkUnit

use of org.apache.gobblin.source.workunit.WorkUnit in project incubator-gobblin by apache.

the class SerializationUtilsTest method testSerializeState.

@Test
public void testSerializeState() throws IOException {
    WorkUnit workUnit1 = WorkUnit.createEmpty();
    workUnit1.setProp("foo", "bar");
    workUnit1.setProp("a", 10);
    SerializationUtils.serializeState(this.fs, new Path(this.outputPath, "wu1"), workUnit1);
    WorkUnit workUnit2 = WorkUnit.createEmpty();
    workUnit2.setProp("foo", "baz");
    workUnit2.setProp("b", 20);
    SerializationUtils.serializeState(this.fs, new Path(this.outputPath, "wu2"), workUnit2);
}

Example 59 with WorkUnit

use of org.apache.gobblin.source.workunit.WorkUnit in project incubator-gobblin by apache.

the class WriterUtilsTest method testGetDefaultWriterFilePathWithWorkUnitState.

@Test
public void testGetDefaultWriterFilePathWithWorkUnitState() {
    String namespace = "gobblin.test";
    String tableName = "test-table";
    SourceState sourceState = new SourceState();
    WorkUnit workUnit = WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName));
    WorkUnitState workUnitState = new WorkUnitState(workUnit);
    Assert.assertEquals(WriterUtils.getWriterFilePath(workUnitState, 0, 0), new Path(workUnitState.getExtract().getOutputFilePath()));
    Assert.assertEquals(WriterUtils.getWriterFilePath(workUnitState, 2, 0), new Path(workUnitState.getExtract().getOutputFilePath(), ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + "0"));
}

Example 60 with WorkUnit

use of org.apache.gobblin.source.workunit.WorkUnit in project incubator-gobblin by apache.

the class WriterUtilsTest method testGetDefaultWriterFilePath.

@Test
public void testGetDefaultWriterFilePath() {
    String namespace = "gobblin.test";
    String tableName = "test-table";
    SourceState sourceState = new SourceState();
    WorkUnit state = WorkUnit.create(new Extract(sourceState, TableType.APPEND_ONLY, namespace, tableName));
    Assert.assertEquals(WriterUtils.getWriterFilePath(state, 0, 0), new Path(state.getExtract().getOutputFilePath()));
    Assert.assertEquals(WriterUtils.getWriterFilePath(state, 2, 0), new Path(state.getExtract().getOutputFilePath(), ConfigurationKeys.DEFAULT_FORK_BRANCH_NAME + "0"));
}