Examples with State org.apache.gobblin.configuration.State used on opensource projects

Example 11 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class DestinationTest method testMethods.

@Test
public void testMethods() {
    State state = new State();
    state.setProp("foo", "bar");
    Destination destination = Destination.of(Destination.DestinationType.HDFS, state);
    Assert.assertEquals(destination.getType(), Destination.DestinationType.HDFS);
    Assert.assertEquals(destination.getProperties().getPropertyNames().size(), 1);
    Assert.assertEquals(destination.getProperties().getProp("foo"), "bar");
}

Example 12 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class PartitionedWriterTest method testWatermarkComputation.

public void testWatermarkComputation(Long committed, Long unacknowledged, Long expected) throws IOException {
    State state = new State();
    state.setProp(ConfigurationKeys.WRITER_PARTITIONER_CLASS, TestPartitioner.class.getCanonicalName());
    String defaultSource = "default";
    WatermarkAwareWriter mockDataWriter = mock(WatermarkAwareWriter.class);
    when(mockDataWriter.isWatermarkCapable()).thenReturn(true);
    when(mockDataWriter.getCommittableWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(committed))));
    when(mockDataWriter.getUnacknowledgedWatermark()).thenReturn(Collections.singletonMap(defaultSource, new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(unacknowledged))));
    PartitionAwareDataWriterBuilder builder = mock(PartitionAwareDataWriterBuilder.class);
    when(builder.validatePartitionSchema(any(Schema.class))).thenReturn(true);
    when(builder.forPartition(any(GenericRecord.class))).thenReturn(builder);
    when(builder.withWriterId(any(String.class))).thenReturn(builder);
    when(builder.build()).thenReturn(mockDataWriter);
    PartitionedDataWriter writer = new PartitionedDataWriter<String, String>(builder, state);
    RecordEnvelope<String> recordEnvelope = new RecordEnvelope<String>("0");
    recordEnvelope.addCallBack(new AcknowledgableWatermark(new DefaultCheckpointableWatermark(defaultSource, new LongWatermark(0))));
    writer.writeEnvelope(recordEnvelope);
    Map<String, CheckpointableWatermark> watermark = writer.getCommittableWatermark();
    System.out.println(watermark.toString());
    if (expected == null) {
        Assert.assertTrue(watermark.isEmpty(), "Expected watermark to be absent");
    } else {
        Assert.assertTrue(watermark.size() == 1);
        Assert.assertEquals((long) expected, ((LongWatermark) watermark.values().iterator().next().getWatermark()).getValue());
    }
}

Example 13 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RetryWriterTest method retryTestNonTransientException.

public void retryTestNonTransientException() throws IOException {
    DataWriter<Void> writer = mock(DataWriter.class);
    doThrow(new NonTransientException()).when(writer).writeEnvelope(any(RecordEnvelope.class));
    DataWriterWrapperBuilder<Void> builder = new DataWriterWrapperBuilder<>(writer, new State());
    DataWriter<Void> retryWriter = builder.build();
    try {
        retryWriter.writeEnvelope(new RecordEnvelope<>(null));
        Assert.fail("Should have failed.");
    } catch (Exception e) {
    }
    verify(writer, atMost(1)).writeEnvelope(any(RecordEnvelope.class));
}

Example 14 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class RetryWriterTest method retryGetFinalState.

public void retryGetFinalState() throws IOException {
    PartitionedDataWriter writer = mock(PartitionedDataWriter.class);
    when(writer.getFinalState()).thenReturn(new State());
    DataWriterWrapperBuilder<Void> builder = new DataWriterWrapperBuilder<>(writer, new State());
    DataWriter<Void> retryWriter = builder.build();
    State state = ((FinalState) retryWriter).getFinalState();
    verify(writer, times(1)).getFinalState();
    Assert.assertTrue(state.contains(RetryWriter.FAILED_WRITES_KEY));
}

Example 15 with State

use of org.apache.gobblin.configuration.State in project incubator-gobblin by apache.

the class SimpleDataWriterTest method setUp.

@BeforeMethod
public void setUp() throws Exception {
    properties = new State();
    // Making the staging and/or output dirs if necessary
    File stagingDir = new File(TestConstants.TEST_STAGING_DIR);
    File outputDir = new File(TestConstants.TEST_OUTPUT_DIR);
    if (!stagingDir.exists()) {
        stagingDir.mkdirs();
    }
    if (!outputDir.exists()) {
        outputDir.mkdirs();
    }
    this.filePath = TestConstants.TEST_EXTRACT_NAMESPACE.replaceAll("\\.", "/") + "/" + TestConstants.TEST_EXTRACT_TABLE + "/" + TestConstants.TEST_EXTRACT_ID + "_" + TestConstants.TEST_EXTRACT_PULL_TYPE;
    properties.setProp(ConfigurationKeys.SIMPLE_WRITER_DELIMITER, "\n");
    properties.setProp(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, TestConstants.TEST_FS_URI);
    properties.setProp(ConfigurationKeys.WRITER_STAGING_DIR, TestConstants.TEST_STAGING_DIR);
    properties.setProp(ConfigurationKeys.WRITER_OUTPUT_DIR, TestConstants.TEST_OUTPUT_DIR);
    properties.setProp(ConfigurationKeys.WRITER_FILE_PATH, this.filePath);
    properties.setProp(ConfigurationKeys.WRITER_FILE_NAME, TestConstants.TEST_FILE_NAME);
    properties.setProp(ConfigurationKeys.SIMPLE_WRITER_PREPEND_SIZE, false);
}