Examples with Extract org.apache.gobblin.source.workunit.Extract used on opensource projects

Example 21 with Extract

use of org.apache.gobblin.source.workunit.Extract in project incubator-gobblin by apache.

the class TestRecordStream method getEmptyTestTaskState.

TaskState getEmptyTestTaskState(String taskId) {
    // Create a TaskState
    WorkUnit workUnit = WorkUnit.create(new Extract(Extract.TableType.SNAPSHOT_ONLY, this.getClass().getName(), this.getClass().getSimpleName()));
    workUnit.setProp(ConfigurationKeys.TASK_KEY_KEY, "taskKey");
    TaskState taskState = new TaskState(new WorkUnitState(workUnit));
    taskState.setProp(ConfigurationKeys.METRICS_ENABLED_KEY, Boolean.toString(false));
    taskState.setTaskId(taskId);
    taskState.setJobId("1234");
    return taskState;
}

Example 22 with Extract

use of org.apache.gobblin.source.workunit.Extract in project incubator-gobblin by apache.

the class SourceState method createExtract.

/**
 * Create a new properly populated {@link Extract} instance.
 *
 * <p>
 *   This method should always return a new unique {@link Extract} instance.
 * </p>
 *
 * @param type {@link org.apache.gobblin.source.workunit.Extract.TableType}
 * @param namespace namespace of the table this extract belongs to
 * @param table name of the table this extract belongs to
 * @return a new unique {@link Extract} instance
 *
 * @Deprecated Use {@link org.apache.gobblin.source.extractor.extract.AbstractSource#createExtract(
 * org.apache.gobblin.source.workunit.Extract.TableType, String, String)}
 */
@Deprecated
public synchronized Extract createExtract(Extract.TableType type, String namespace, String table) {
    Extract extract = new Extract(this, type, namespace, table);
    while (EXTRACT_SET.contains(extract)) {
        if (Strings.isNullOrEmpty(extract.getExtractId())) {
            extract.setExtractId(DTF.print(new DateTime()));
        } else {
            DateTime extractDateTime = DTF.parseDateTime(extract.getExtractId());
            extract.setExtractId(DTF.print(extractDateTime.plusSeconds(1)));
        }
    }
    EXTRACT_SET.add(extract);
    return extract;
}

Example 23 with Extract

use of org.apache.gobblin.source.workunit.Extract in project incubator-gobblin by apache.

the class GoogleWebmasterExtractorTest method getWorkUnitState1.

public static WorkUnitState getWorkUnitState1() {
    WorkUnit wu = new WorkUnit(new Extract(Extract.TableType.APPEND_ONLY, "namespace", "table"));
    wu.setWatermarkInterval(new WatermarkInterval(new LongWatermark(20160101235959L), new LongWatermark(20160102235959L)));
    State js = new State();
    return new WorkUnitState(wu, js);
}

Example 24 with Extract

use of org.apache.gobblin.source.workunit.Extract in project incubator-gobblin by apache.

the class WorkUnitStateTest method WorkUnitState.

@Test
public void WorkUnitState() throws IOException {
    SourceState sourceState = new SourceState();
    Extract extract = sourceState.createExtract(Extract.TableType.APPEND_ONLY, "test", "test");
    WorkUnit wu = sourceState.createWorkUnit(extract);
    WorkUnitState state = new WorkUnitState(wu);
    Assert.assertEquals(state.getProp("string", "some string"), "some string");
    Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(0), "item1");
    Assert.assertEquals(state.getPropAsList("list", "item1,item2").get(1), "item2");
    Assert.assertEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE);
    Assert.assertEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE);
    Assert.assertEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE);
    Assert.assertEquals(state.getPropAsBoolean("boolean", true), true);
    wu.setProp("string", "some string");
    wu.setProp("list", "item1,item2");
    wu.setProp("long", Long.MAX_VALUE);
    wu.setProp("int", Integer.MAX_VALUE);
    wu.setProp("double", Double.MAX_VALUE);
    wu.setProp("boolean", true);
    state = new WorkUnitState(wu);
    Assert.assertEquals(state.getProp("string"), "some string");
    Assert.assertEquals(state.getPropAsList("list").get(0), "item1");
    Assert.assertEquals(state.getPropAsList("list").get(1), "item2");
    Assert.assertEquals(state.getPropAsLong("long"), Long.MAX_VALUE);
    Assert.assertEquals(state.getPropAsInt("int"), Integer.MAX_VALUE);
    Assert.assertEquals(state.getPropAsDouble("double"), Double.MAX_VALUE);
    Assert.assertEquals(state.getPropAsBoolean("boolean"), true);
    state.setProp("string", "some other string");
    state.setProp("list", "item3,item4");
    state.setProp("long", Long.MIN_VALUE);
    state.setProp("int", Integer.MIN_VALUE);
    state.setProp("double", Double.MIN_VALUE);
    state.setProp("boolean", false);
    Assert.assertNotEquals(state.getProp("string", "some string"), "some string");
    Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(0), "item1");
    Assert.assertNotEquals(state.getPropAsList("list", "item1,item2").get(1), "item2");
    Assert.assertNotEquals(state.getPropAsLong("long", Long.MAX_VALUE), Long.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsInt("int", Integer.MAX_VALUE), Integer.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsDouble("double", Double.MAX_VALUE), Double.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsBoolean("boolean", true), true);
    Assert.assertNotEquals(state.getProp("string"), "some string");
    Assert.assertNotEquals(state.getPropAsList("list").get(0), "item1");
    Assert.assertNotEquals(state.getPropAsList("list").get(1), "item2");
    Assert.assertNotEquals(state.getPropAsLong("long"), Long.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsInt("int"), Integer.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsDouble("double"), Double.MAX_VALUE);
    Assert.assertNotEquals(state.getPropAsBoolean("boolean"), true);
    Assert.assertEquals(state.getProp("string"), "some other string");
    Assert.assertEquals(state.getPropAsList("list").get(0), "item3");
    Assert.assertEquals(state.getPropAsList("list").get(1), "item4");
    Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE);
    Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE);
    Assert.assertEquals(state.getPropAsDouble("double"), Double.MIN_VALUE);
    Assert.assertEquals(state.getPropAsBoolean("boolean"), false);
    ByteArrayOutputStream byteStream = new ByteArrayOutputStream(1024);
    DataOutputStream out = new DataOutputStream(byteStream);
    state.write(out);
    DataInputStream in = new DataInputStream(new ByteArrayInputStream(byteStream.toByteArray()));
    state = new WorkUnitState();
    Assert.assertEquals(state.getProp("string"), null);
    Assert.assertEquals(state.getProp("list"), null);
    Assert.assertEquals(state.getProp("long"), null);
    Assert.assertEquals(state.getProp("int"), null);
    Assert.assertEquals(state.getProp("double"), null);
    Assert.assertEquals(state.getProp("boolean"), null);
    state.readFields(in);
    Assert.assertEquals(state.getProp("string"), "some other string");
    Assert.assertEquals(state.getPropAsList("list").get(0), "item3");
    Assert.assertEquals(state.getPropAsList("list").get(1), "item4");
    Assert.assertEquals(state.getPropAsLong("long"), Long.MIN_VALUE);
    Assert.assertEquals(state.getPropAsInt("int"), Integer.MIN_VALUE);
    Assert.assertEquals(state.getPropAsDouble("double"), Double.MIN_VALUE);
    Assert.assertEquals(state.getPropAsBoolean("boolean"), false);
    WorkUnitState state2 = new WorkUnitState();
    state2.addAll(state);
    Assert.assertEquals(state2.getProp("string"), "some other string");
    Assert.assertEquals(state2.getPropAsList("list").get(0), "item3");
    Assert.assertEquals(state2.getPropAsList("list").get(1), "item4");
    Assert.assertEquals(state2.getPropAsLong("long"), Long.MIN_VALUE);
    Assert.assertEquals(state2.getPropAsInt("int"), Integer.MIN_VALUE);
    Assert.assertEquals(state2.getPropAsDouble("double"), Double.MIN_VALUE);
    Assert.assertEquals(state2.getPropAsBoolean("boolean"), false);
}

Example 25 with Extract

use of org.apache.gobblin.source.workunit.Extract in project incubator-gobblin by apache.

the class JsonElementConversionFactoryTest method setUp.

@BeforeClass
public static void setUp() {
    WorkUnit workUnit = new WorkUnit(new SourceState(), new Extract(new SourceState(), Extract.TableType.SNAPSHOT_ONLY, "namespace", "dummy_table"));
    state = new WorkUnitState(workUnit);
    Type listType = new TypeToken<JsonObject>() {
    }.getType();
    Gson gson = new Gson();
    testData = gson.fromJson(new InputStreamReader(JsonElementConversionFactoryTest.class.getResourceAsStream("/converter/JsonElementConversionFactoryTest.json")), listType);
}