Examples with RawMessage org.apache.pulsar.client.api.RawMessage used on opensource projects

Example 1 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class RawReaderTest method testFlowControl.

/**
 * Try to fill the receiver queue, and drain it multiple times
 */
@Test
public void testFlowControl() throws Exception {
    int numMessages = RawReaderImpl.DEFAULT_RECEIVER_QUEUE_SIZE * 5;
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    publishMessages(topic, numMessages);
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    List<Future<RawMessage>> futures = new ArrayList<>();
    Set<String> keys = new HashSet<>();
    // +1 to make sure we read past the end
    for (int i = 0; i < numMessages + 1; i++) {
        futures.add(reader.readNextAsync());
    }
    int timeouts = 0;
    for (Future<RawMessage> f : futures) {
        try (RawMessage m = f.get(1, TimeUnit.SECONDS)) {
            // Assert each key is unique
            Assert.assertTrue(keys.add(extractKey(m)));
        } catch (TimeoutException te) {
            timeouts++;
        }
    }
    Assert.assertEquals(timeouts, 1);
    Assert.assertEquals(keys.size(), numMessages);
}

Example 2 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class RawReaderTest method testBatchingExtractKeysAndIds.

@Test
public void testBatchingExtractKeysAndIds() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-content-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-content-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key3").setContent("my-content-3".getBytes()).build()).get();
    }
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    try (RawMessage m = reader.readNextAsync().get()) {
        List<ImmutablePair<MessageId, String>> idsAndKeys = RawBatchConverter.extractIdsAndKeys(m);
        Assert.assertEquals(idsAndKeys.size(), 3);
        // assert message ids are in correct order
        Assert.assertTrue(idsAndKeys.get(0).getLeft().compareTo(idsAndKeys.get(1).getLeft()) < 0);
        Assert.assertTrue(idsAndKeys.get(1).getLeft().compareTo(idsAndKeys.get(2).getLeft()) < 0);
        // assert keys are as expected
        Assert.assertEquals(idsAndKeys.get(0).getRight(), "key1");
        Assert.assertEquals(idsAndKeys.get(1).getRight(), "key2");
        Assert.assertEquals(idsAndKeys.get(2).getRight(), "key3");
    } finally {
        reader.closeAsync().get();
    }
}

Example 3 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class CompactedTopicTest method buildCompactedLedger.

/**
 * Build a compacted ledger, and return the id of the ledger, the position of the different
 * entries in the ledger, and a list of gaps, and the entry which should be returned after the gap.
 */
private Triple<Long, List<Pair<MessageIdData, Long>>, List<Pair<MessageIdData, Long>>> buildCompactedLedger(BookKeeper bk, int count) throws Exception {
    LedgerHandle lh = bk.createLedger(1, 1, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    List<Pair<MessageIdData, Long>> positions = new ArrayList<>();
    List<Pair<MessageIdData, Long>> idsInGaps = new ArrayList<>();
    AtomicLong ledgerIds = new AtomicLong(10L);
    AtomicLong entryIds = new AtomicLong(0L);
    CompletableFuture.allOf(IntStream.range(0, count).mapToObj((i) -> {
        List<MessageIdData> idsInGap = new ArrayList<MessageIdData>();
        if (r.nextInt(10) == 1) {
            long delta = r.nextInt(10) + 1;
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
            ledgerIds.addAndGet(delta);
            entryIds.set(0);
        }
        long delta = r.nextInt(5);
        if (delta != 0) {
            idsInGap.add(MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.get() + 1).build());
        }
        MessageIdData id = MessageIdData.newBuilder().setLedgerId(ledgerIds.get()).setEntryId(entryIds.addAndGet(delta + 1)).build();
        @Cleanup RawMessage m = new RawMessageImpl(id, Unpooled.EMPTY_BUFFER);
        CompletableFuture<Void> f = new CompletableFuture<>();
        ByteBuf buffer = m.serialize();
        lh.asyncAddEntry(buffer, (rc, ledger, eid, ctx) -> {
            if (rc != BKException.Code.OK) {
                f.completeExceptionally(BKException.create(rc));
            } else {
                positions.add(Pair.of(id, eid));
                idsInGap.forEach((gid) -> idsInGaps.add(Pair.of(gid, eid)));
                f.complete(null);
            }
        }, null);
        buffer.release();
        return f;
    }).toArray(CompletableFuture[]::new)).get();
    lh.close();
    return Triple.of(lh.getId(), positions, idsInGaps);
}

Example 4 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class ChecksumTest method verifyChecksumSentToConsumer.

@Test
public void verifyChecksumSentToConsumer() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-1";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    RawReader reader = RawReader.create(pulsarClient, topicName, "sub").get();
    producer.send("Hello".getBytes());
    RawMessage msg = reader.readNextAsync().get();
    ByteBuf b = msg.getHeadersAndPayload();
    assertTrue(Commands.hasChecksum(b));
    int parsedChecksum = Commands.readChecksum(b);
    int computedChecksum = Crc32cIntChecksum.computeChecksum(b);
    assertEquals(parsedChecksum, computedChecksum);
    producer.close();
    reader.closeAsync().get();
}

Example 5 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class CompactedTopicImpl method readOneMessageId.

private static CompletableFuture<MessageIdData> readOneMessageId(LedgerHandle lh, long entryId) {
    CompletableFuture<MessageIdData> promise = new CompletableFuture<>();
    lh.asyncReadEntries(entryId, entryId, (rc, _lh, seq, ctx) -> {
        if (rc != BKException.Code.OK) {
            promise.completeExceptionally(BKException.create(rc));
        } else {
            try (RawMessage m = RawMessageImpl.deserializeFrom(seq.nextElement().getEntryBuffer())) {
                promise.complete(m.getMessageIdData());
            } catch (NoSuchElementException e) {
                log.error("No such entry {} in ledger {}", entryId, lh.getId());
                promise.completeExceptionally(e);
            }
        }
    }, null);
    return promise;
}