Examples with WriteResponse org.apache.gobblin.writer.WriteResponse used on opensource projects

Example 1 with WriteResponse

use of org.apache.gobblin.writer.WriteResponse in project incubator-gobblin by apache.

the class Kafka09DataWriterTest method testStringSerialization.

@Test
public void testStringSerialization() throws IOException, InterruptedException, ExecutionException {
    String topic = "testStringSerialization08";
    _kafkaTestHelper.provisionTopic(topic);
    Properties props = new Properties();
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, topic);
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", "localhost:" + _kafkaTestHelper.getKafkaServerPort());
    props.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    Kafka09DataWriter<String> kafka09DataWriter = new Kafka09DataWriter<String>(props);
    String messageString = "foobar";
    WriteCallback callback = mock(WriteCallback.class);
    Future<WriteResponse> future;
    try {
        future = kafka09DataWriter.write(messageString, callback);
        kafka09DataWriter.flush();
        verify(callback, times(1)).onSuccess(isA(WriteResponse.class));
        verify(callback, never()).onFailure(isA(Exception.class));
        Assert.assertTrue(future.isDone(), "Future should be done");
        System.out.println(future.get().getStringResponse());
        byte[] message = _kafkaTestHelper.getIteratorForTopic(topic).next().message();
        String messageReceived = new String(message);
        Assert.assertEquals(messageReceived, messageString);
    } finally {
        kafka09DataWriter.close();
    }
}

Example 2 with WriteResponse

use of org.apache.gobblin.writer.WriteResponse in project incubator-gobblin by apache.

the class StreamingKafkaSpecExecutorTest method testAddSpec.

@Test
public void testAddSpec() throws Exception {
    _closer = Closer.create();
    _properties = new Properties();
    // Properties for Producer
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_TOPIC, TOPIC);
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "bootstrap.servers", _kafkaBrokers);
    _properties.setProperty(KafkaWriterConfigurationKeys.KAFKA_PRODUCER_CONFIG_PREFIX + "value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    // Properties for Consumer
    _properties.setProperty("jobSpecMonitor.kafka.zookeeper.connect", zkConnect);
    _properties.setProperty(SimpleKafkaSpecExecutor.SPEC_KAFKA_TOPICS_KEY, TOPIC);
    _properties.setProperty("gobblin.cluster.jobconf.fullyQualifiedPath", _JOBS_DIR_PATH);
    Config config = ConfigUtils.propertiesToConfig(_properties);
    // SEI Producer
    _seip = _closer.register(new SimpleKafkaSpecProducer(config));
    String addedSpecUriString = "/foo/bar/addedSpec";
    Spec spec = initJobSpec(addedSpecUriString);
    WriteResponse writeResponse = (WriteResponse) _seip.addSpec(spec).get();
    log.info("WriteResponse: " + writeResponse);
    _jobCatalog = new NonObservingFSJobCatalog(config.getConfig("gobblin.cluster"));
    _jobCatalog.startAsync().awaitRunning();
    // SEI Consumer
    _seic = _closer.register(new StreamingKafkaSpecConsumer(config, _jobCatalog));
    _seic.startAsync().awaitRunning();
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.ADD), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(addedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 3 with WriteResponse

use of org.apache.gobblin.writer.WriteResponse in project incubator-gobblin by apache.

the class StreamingKafkaSpecExecutorTest method testDeleteSpec.

@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
    // delete needs to be on a job spec that exists to get notification
    String deletedSpecUriString = "/foo/bar/addedSpec";
    WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
    log.info("WriteResponse: " + writeResponse);
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 4 with WriteResponse

use of org.apache.gobblin.writer.WriteResponse in project incubator-gobblin by apache.

the class SimpleKafkaSpecExecutorTest method testDeleteSpec.

@Test(dependsOnMethods = "testUpdateSpec")
public void testDeleteSpec() throws Exception {
    String deletedSpecUriString = "/foo/bar/deletedSpec";
    WriteResponse writeResponse = (WriteResponse) _seip.deleteSpec(new URI(deletedSpecUriString)).get();
    log.info("WriteResponse: " + writeResponse);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.DELETE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(deletedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}

Example 5 with WriteResponse

use of org.apache.gobblin.writer.WriteResponse in project incubator-gobblin by apache.

the class SimpleKafkaSpecExecutorTest method testUpdateSpec.

@Test(dependsOnMethods = "testAddSpec")
public void testUpdateSpec() throws Exception {
    String updatedSpecUriString = "/foo/bar/updatedSpec";
    Spec spec = initJobSpec(updatedSpecUriString);
    WriteResponse writeResponse = (WriteResponse) _seip.updateSpec(spec).get();
    log.info("WriteResponse: " + writeResponse);
    try {
        Thread.sleep(1000);
    } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
    }
    List<Pair<SpecExecutor.Verb, Spec>> consumedEvent = _seic.changedSpecs().get();
    Assert.assertTrue(consumedEvent.size() == 1, "Consumption did not match production");
    Map.Entry<SpecExecutor.Verb, Spec> consumedSpecAction = consumedEvent.get(0);
    Assert.assertTrue(consumedSpecAction.getKey().equals(SpecExecutor.Verb.UPDATE), "Verb did not match");
    Assert.assertTrue(consumedSpecAction.getValue().getUri().toString().equals(updatedSpecUriString), "Expected URI did not match");
    Assert.assertTrue(consumedSpecAction.getValue() instanceof JobSpec, "Expected JobSpec");
}