Examples with WriteCallback org.apache.gobblin.writer.WriteCallback used on opensource projects

Example 1 with WriteCallback

use of org.apache.gobblin.writer.WriteCallback in project incubator-gobblin by apache.

the class Kafka09DataWriterTest method testBinarySerialization.

@Test
public void testBinarySerialization() throws IOException, InterruptedException {
    String topic = "testBinarySerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    Kafka09DataWriter<byte[]> kafka09DataWriter = new Kafka09DataWriter<byte[]>(props);
    WriteCallback callback = mock(WriteCallback.class);
    byte[] messageBytes = TestUtils.generateRandomBytes();
    try {
        kafka09DataWriter.write(messageBytes, callback);
    } finally {
        kafka09DataWriter.close();
    }
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
    Assert.assertEquals(message, messageBytes);
}

Example 2 with WriteCallback

use of org.apache.gobblin.writer.WriteCallback in project incubator-gobblin by apache.

the class Kafka09DataWriterTest method testAvroSerialization.

@Test
public void testAvroSerialization() throws IOException, InterruptedException, SchemaRegistryException {
    String topic = "testAvroSerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", LiAvroSerializer.class.getName());
    // set up mock schema registry
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + KafkaSchemaRegistryConfigurationKeys.KAFKA_SCHEMA_REGISTRY_CLASS, ConfigDrivenMd5SchemaRegistry.class.getCanonicalName());
    Kafka09DataWriter<GenericRecord> kafka09DataWriter = new Kafka09DataWriter<>(props);
    WriteCallback callback = mock(WriteCallback.class);
    GenericRecord record = TestUtils.generateRandomAvroRecord();
    try {
        kafka09DataWriter.write(record, callback);
    } finally {
        kafka09DataWriter.close();
    }
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
    ConfigDrivenMd5SchemaRegistry schemaReg = new ConfigDrivenMd5SchemaRegistry(topic, record.getSchema());
    LiAvroDeserializer deser = new LiAvroDeserializer(schemaReg);
    GenericRecord receivedRecord = deser.deserialize(topic, message);
    Assert.assertEquals(record.toString(), receivedRecord.toString());
}

Example 3 with WriteCallback

use of org.apache.gobblin.writer.WriteCallback in project incubator-gobblin by apache.

the class Kafka09DataWriterTest method testStringSerialization.

@Test
public void testStringSerialization() throws IOException, InterruptedException, ExecutionException {
    String topic = "testStringSerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    String messageString = "foobar";
    WriteCallback callback = mock(WriteCallback.class);
    Future<WriteResponse> future;
    try {
        future = kafka09DataWriter.write(messageString, callback);
        kafka09DataWriter.flush();
        verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
        verify(callback, never()).onFailure(isA(Exception.class));
        Assert.assertTrue(future.isDone(), "Future should be done");
        System.out.println(future.get().getStringResponse());
        byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
        String messageReceived = new String(message);
        Assert.assertEquals(messageReceived, messageString);
    } finally {
        kafka09DataWriter.close();
    }
}

Example 4 with WriteCallback

use of org.apache.gobblin.writer.WriteCallback in project incubator-gobblin by apache.

the class Kafka08DataWriterTest method testBinarySerialization.

@Test
public void testBinarySerialization() throws IOException, InterruptedException {
    String topic = "testBinarySerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    Kafka08DataWriter<byte[]> kafka08DataWriter = new Kafka08DataWriter<byte[]>(props);
    WriteCallback callback = mock(WriteCallback.class);
    byte[] messageBytes = TestUtils.generateRandomBytes();
    try {
        kafka08DataWriter.write(messageBytes, callback);
    } finally {
        kafka08DataWriter.close();
    }
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
    Assert.assertEquals(message, messageBytes);
}

Example 5 with WriteCallback

use of org.apache.gobblin.writer.WriteCallback in project incubator-gobblin by apache.

the class EventhubDataWriterTest method testSingleBatch.

@Test
public void testSingleBatch() {
    // mock eventhub data writer
    Properties props = new Properties();
    EventhubDataWriter eventhubDataWriter = Mockito.spy(new EventhubDataWriter(props, mockHttpClient));
    Mockito.doNothing().when(eventhubDataWriter).refreshSignature();
    List<String> records = new LinkedList<>();
    for (int i = 0; i < 50; ++i) records.add(new String("abcdefgh"));
    Batch<String> batch = mock(Batch.class);
    WriteCallback callback = mock(WriteCallback.class);
    Mockito.when(batch.getRecords()).thenReturn(records);
    Future<WriteResponse> future = eventhubDataWriter.write(batch, callback);
    verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
    verify(callback, never()).onFailure(isA(Exception.class));
    Assert.assertTrue(future.isDone(), "Future should be done");
}