Examples with PulsarKafkaConsumer org.apache.kafka.clients.consumer.PulsarKafkaConsumer used on opensource projects

Example 1 with PulsarKafkaConsumer

use of org.apache.kafka.clients.consumer.PulsarKafkaConsumer in project incubator-pulsar by apache.

the class KafkaConsumerTest method testConsumerAutoCommit.

@Test
public void testConsumerAutoCommit() throws Exception {
    String topic = "persistent://sample/standalone/ns/testConsumerAutoCommit";
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "true");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).create();
    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    AtomicInteger received = new AtomicInteger();
    while (received.get() < 10) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        records.forEach(record -> {
            assertEquals(record.key(), Integer.toString(received.get()));
            assertEquals(record.value(), "hello-" + received.get());
            received.incrementAndGet();
        });
    }
    consumer.close();
    // Re-open consumer and verify every message was acknowledged
    Consumer<String, String> consumer2 = new PulsarKafkaConsumer<>(props);
    consumer2.subscribe(Arrays.asList(topic));
    ConsumerRecords<String, String> records = consumer2.poll(100);
    assertEquals(records.count(), 0);
    consumer2.close();
}

Example 2 with PulsarKafkaConsumer

use of org.apache.kafka.clients.consumer.PulsarKafkaConsumer in project incubator-pulsar by apache.

the class KafkaConsumerTest method testConsumerSeekToEnd.

@Test
public void testConsumerSeekToEnd() throws Exception {
    String topic = "persistent://sample/standalone/ns/testSimpleConsumer";
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "false");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).create();
    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    AtomicInteger received = new AtomicInteger();
    while (received.get() < 10) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        records.forEach(record -> {
            assertEquals(record.key(), Integer.toString(received.get()));
            assertEquals(record.value(), "hello-" + received.get());
            received.incrementAndGet();
        });
        consumer.commitSync();
    }
    consumer.seekToEnd(Collections.emptyList());
    Thread.sleep(500);
    consumer.close();
    // Recreate the consumer
    consumer = new PulsarKafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));
    ConsumerRecords<String, String> records = consumer.poll(100);
    // Since we are at the end of the topic, there should be no messages
    assertEquals(records.count(), 0);
    consumer.close();
}

Example 3 with PulsarKafkaConsumer

use of org.apache.kafka.clients.consumer.PulsarKafkaConsumer in project incubator-pulsar by apache.

the class KafkaConsumerTest method testPartitions.

@Test
public void testPartitions() throws Exception {
    String topic = "persistent://sample/standalone/ns/testPartitions";
    // Create 8 partitions in topic
    admin.properties().createProperty("sample", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topic, 8);
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "true");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // Create 2 Kakfa consumer and verify each gets half of the messages
    List<Consumer<String, String>> consumers = new ArrayList<>();
    for (int c = 0; c < 2; c++) {
        Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList(topic));
        consumers.add(consumer);
    }
    int N = 8 * 3;
    for (int i = 0; i < N; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    consumers.forEach(consumer -> {
        int expectedMessaged = N / consumers.size();
        for (int i = 0; i < expectedMessaged; ) {
            ConsumerRecords<String, String> records = consumer.poll(100);
            i += records.count();
        }
        // No more messages for this consumer
        ConsumerRecords<String, String> records = consumer.poll(100);
        assertEquals(records.count(), 0);
    });
    consumers.forEach(Consumer::close);
}

Example 4 with PulsarKafkaConsumer

use of org.apache.kafka.clients.consumer.PulsarKafkaConsumer in project incubator-pulsar by apache.

the class KafkaConsumerTest method testConsumerManualOffsetCommit.

@Test
public void testConsumerManualOffsetCommit() throws Exception {
    String topic = "persistent://sample/standalone/ns/testConsumerManualOffsetCommit";
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "false");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).create();
    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    AtomicInteger received = new AtomicInteger();
    while (received.get() < 10) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        records.forEach(record -> {
            assertEquals(record.key(), Integer.toString(received.get()));
            assertEquals(record.value(), "hello-" + received.get());
            Map<TopicPartition, OffsetAndMetadata> offsets = new HashMap<>();
            offsets.put(new TopicPartition(record.topic(), record.partition()), new OffsetAndMetadata(record.offset()));
            consumer.commitSync(offsets);
            received.incrementAndGet();
        });
    }
    consumer.close();
    // Re-open consumer and verify every message was acknowledged
    Consumer<String, String> consumer2 = new PulsarKafkaConsumer<>(props);
    consumer2.subscribe(Arrays.asList(topic));
    ConsumerRecords<String, String> records = consumer2.poll(100);
    assertEquals(records.count(), 0);
    consumer2.close();
}

Example 5 with PulsarKafkaConsumer

use of org.apache.kafka.clients.consumer.PulsarKafkaConsumer in project incubator-pulsar by apache.

the class KafkaConsumerTest method testConsumerSeek.

@Test
public void testConsumerSeek() throws Exception {
    String topic = "persistent://sample/standalone/ns/testSimpleConsumer";
    Properties props = new Properties();
    props.put("bootstrap.servers", lookupUrl.toString());
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "false");
    props.put("key.deserializer", StringDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());
    Consumer<String, String> consumer = new PulsarKafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));
    Producer<byte[]> pulsarProducer = pulsarClient.newProducer().topic(topic).create();
    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = MessageBuilder.create().setKey(Integer.toString(i)).setContent(("hello-" + i).getBytes()).build();
        pulsarProducer.send(msg);
    }
    AtomicInteger received = new AtomicInteger();
    while (received.get() < 10) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        records.forEach(record -> {
            assertEquals(record.key(), Integer.toString(received.get()));
            assertEquals(record.value(), "hello-" + received.get());
            received.incrementAndGet();
        });
        consumer.commitSync();
    }
    consumer.seekToBeginning(Collections.emptyList());
    Thread.sleep(500);
    // Messages should be available again
    received.set(0);
    while (received.get() < 10) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        records.forEach(record -> {
            assertEquals(record.key(), Integer.toString(received.get()));
            assertEquals(record.value(), "hello-" + received.get());
            received.incrementAndGet();
        });
        consumer.commitSync();
    }
    consumer.close();
}