Examples with RawReader org.apache.pulsar.client.api.RawReader used on opensource projects

Example 1 with RawReader

use of org.apache.pulsar.client.api.RawReader in project incubator-pulsar by apache.

the class RawReaderTest method testReadCancellationOnClose.

@Test
public void testReadCancellationOnClose() throws Exception {
    int numKeys = 10;
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    publishMessages(topic, numKeys / 2);
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    List<Future<RawMessage>> futures = new ArrayList<>();
    for (int i = 0; i < numKeys; i++) {
        futures.add(reader.readNextAsync());
    }
    for (int i = 0; i < numKeys / 2; i++) {
        // complete successfully
        futures.remove(0).get();
    }
    reader.closeAsync().get();
    while (!futures.isEmpty()) {
        try {
            futures.remove(0).get();
            Assert.fail("Should have been cancelled");
        } catch (CancellationException ee) {
        // correct behaviour
        }
    }
}

Example 2 with RawReader

use of org.apache.pulsar.client.api.RawReader in project incubator-pulsar by apache.

the class RawReaderTest method testFlowControl.

/**
 * Try to fill the receiver queue, and drain it multiple times
 */
@Test
public void testFlowControl() throws Exception {
    int numMessages = RawReaderImpl.DEFAULT_RECEIVER_QUEUE_SIZE * 5;
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    publishMessages(topic, numMessages);
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    List<Future<RawMessage>> futures = new ArrayList<>();
    Set<String> keys = new HashSet<>();
    // +1 to make sure we read past the end
    for (int i = 0; i < numMessages + 1; i++) {
        futures.add(reader.readNextAsync());
    }
    int timeouts = 0;
    for (Future<RawMessage> f : futures) {
        try (RawMessage m = f.get(1, TimeUnit.SECONDS)) {
            // Assert each key is unique
            Assert.assertTrue(keys.add(extractKey(m)));
        } catch (TimeoutException te) {
            timeouts++;
        }
    }
    Assert.assertEquals(timeouts, 1);
    Assert.assertEquals(keys.size(), numMessages);
}

Example 3 with RawReader

use of org.apache.pulsar.client.api.RawReader in project incubator-pulsar by apache.

the class RawReaderTest method testBatchingExtractKeysAndIds.

@Test
public void testBatchingExtractKeysAndIds() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-content-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-content-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key3").setContent("my-content-3".getBytes()).build()).get();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m = reader.readNextAsync().get()) {
        List<ImmutablePair<MessageId, String>> idsAndKeys = RawBatchConverter.extractIdsAndKeys(m);
        Assert.assertEquals(idsAndKeys.size(), 3);
        // assert message ids are in correct order
        Assert.assertTrue(idsAndKeys.get(0).getLeft().compareTo(idsAndKeys.get(1).getLeft()) < 0);
        Assert.assertTrue(idsAndKeys.get(1).getLeft().compareTo(idsAndKeys.get(2).getLeft()) < 0);
        // assert keys are as expected
        Assert.assertEquals(idsAndKeys.get(0).getRight(), "key1");
        Assert.assertEquals(idsAndKeys.get(1).getRight(), "key2");
        Assert.assertEquals(idsAndKeys.get(2).getRight(), "key3");
    } finally {
        reader.closeAsync().get();
    }
}

Example 4 with RawReader

use of org.apache.pulsar.client.api.RawReader in project incubator-pulsar by apache.

the class ChecksumTest method verifyChecksumSentToConsumer.

@Test
public void verifyChecksumSentToConsumer() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-1";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    RawReader reader = RawReader.create(pulsarClient, topicName, "sub").get();
    producer.send("Hello".getBytes());
    RawMessage msg = reader.readNextAsync().get();
    ByteBuf b = msg.getHeadersAndPayload();
    assertTrue(Commands.hasChecksum(b));
    int parsedChecksum = Commands.readChecksum(b);
    int computedChecksum = Crc32cIntChecksum.computeChecksum(b);
    assertEquals(parsedChecksum, computedChecksum);
    producer.close();
    reader.closeAsync().get();
}

Example 5 with RawReader

use of org.apache.pulsar.client.api.RawReader in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseTwoLoop.

private void phaseTwoLoop(RawReader reader, MessageId to, Map<String, MessageId> latestForKey, LedgerHandle lh, Semaphore outstanding, CompletableFuture<Void> promise) {
    reader.readNextAsync().whenCompleteAsync((m, exception) -> {
        if (exception != null) {
            promise.completeExceptionally(exception);
            return;
        } else if (promise.isDone()) {
            return;
        }
        MessageId id = m.getMessageId();
        Optional<RawMessage> messageToAdd = Optional.empty();
        if (RawBatchConverter.isBatch(m)) {
            try {
                messageToAdd = RawBatchConverter.rebatchMessage(m, (key, subid) -> latestForKey.get(key).equals(subid));
            } catch (IOException ioe) {
                log.info("Error decoding batch for message {}. Whole batch will be included in output", id, ioe);
                messageToAdd = Optional.of(m);
            }
        } else {
            String key = extractKey(m);
            if (latestForKey.get(key).equals(id)) {
                messageToAdd = Optional.of(m);
            } else {
                m.close();
            }
        }
        messageToAdd.ifPresent((toAdd) -> {
            try {
                outstanding.acquire();
                CompletableFuture<Void> addFuture = addToCompactedLedger(lh, toAdd).whenComplete((res, exception2) -> {
                    outstanding.release();
                    if (exception2 != null) {
                        promise.completeExceptionally(exception2);
                    }
                });
                if (to.equals(id)) {
                    addFuture.whenComplete((res, exception2) -> {
                        if (exception2 == null) {
                            promise.complete(null);
                        }
                    });
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                promise.completeExceptionally(ie);
            }
        });
        phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise);
    }, scheduler);
}