Examples with RawMessage org.apache.pulsar.client.api.RawMessage used on opensource projects

Example 6 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class TwoPhaseCompactor method phaseTwoLoop.

private void phaseTwoLoop(RawReader reader, MessageId to, Map<String, MessageId> latestForKey, LedgerHandle lh, Semaphore outstanding, CompletableFuture<Void> promise) {
    reader.readNextAsync().whenCompleteAsync((m, exception) -> {
        if (exception != null) {
            promise.completeExceptionally(exception);
            return;
        } else if (promise.isDone()) {
            return;
        }
        MessageId id = m.getMessageId();
        Optional<RawMessage> messageToAdd = Optional.empty();
        if (RawBatchConverter.isBatch(m)) {
            try {
                messageToAdd = RawBatchConverter.rebatchMessage(m, (key, subid) -> latestForKey.get(key).equals(subid));
            } catch (IOException ioe) {
                log.info("Error decoding batch for message {}. Whole batch will be included in output", id, ioe);
                messageToAdd = Optional.of(m);
            }
        } else {
            String key = extractKey(m);
            if (latestForKey.get(key).equals(id)) {
                messageToAdd = Optional.of(m);
            } else {
                m.close();
            }
        }
        messageToAdd.ifPresent((toAdd) -> {
            try {
                outstanding.acquire();
                CompletableFuture<Void> addFuture = addToCompactedLedger(lh, toAdd).whenComplete((res, exception2) -> {
                    outstanding.release();
                    if (exception2 != null) {
                        promise.completeExceptionally(exception2);
                    }
                });
                if (to.equals(id)) {
                    addFuture.whenComplete((res, exception2) -> {
                        if (exception2 == null) {
                            promise.complete(null);
                        }
                    });
                }
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
                promise.completeExceptionally(ie);
            }
        });
        phaseTwoLoop(reader, to, latestForKey, lh, outstanding, promise);
    }, scheduler);
}

Example 7 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class ReplicatorTest method verifyChecksumAfterReplication.

@Test(timeOut = 30000)
public void verifyChecksumAfterReplication() throws Exception {
    final String topicName = "persistent://pulsar/global/ns/checksumAfterReplication";
    PulsarClient c1 = PulsarClient.builder().serviceUrl(url1.toString()).build();
    Producer<byte[]> p1 = c1.newProducer().topic(topicName).create();
    PulsarClient c2 = PulsarClient.builder().serviceUrl(url2.toString()).build();
    RawReader reader2 = RawReader.create(c2, topicName, "sub").get();
    p1.send("Hello".getBytes());
    RawMessage msg = reader2.readNextAsync().get();
    ByteBuf b = msg.getHeadersAndPayload();
    assertTrue(Commands.hasChecksum(b));
    int parsedChecksum = Commands.readChecksum(b);
    int computedChecksum = Crc32cIntChecksum.computeChecksum(b);
    assertEquals(parsedChecksum, computedChecksum);
    p1.close();
    reader2.closeAsync().get();
}

Example 8 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class CompactorTest method compactAndVerify.

private List<String> compactAndVerify(String topic, Map<String, byte[]> expected) throws Exception {
    BookKeeper bk = pulsar.getBookKeeperClientFactory().create(this.conf, null);
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    long compactedLedgerId = compactor.compact(topic).get();
    LedgerHandle ledger = bk.openLedger(compactedLedgerId, Compactor.COMPACTED_TOPIC_LEDGER_DIGEST_TYPE, Compactor.COMPACTED_TOPIC_LEDGER_PASSWORD);
    // 0..lac
    Assert.assertEquals(// 0..lac
    ledger.getLastAddConfirmed() + 1, expected.size(), "Should have as many entries as there is keys");
    List<String> keys = new ArrayList<>();
    Enumeration<LedgerEntry> entries = ledger.readEntries(0, ledger.getLastAddConfirmed());
    while (entries.hasMoreElements()) {
        ByteBuf buf = entries.nextElement().getEntryBuffer();
        RawMessage m = RawMessageImpl.deserializeFrom(buf);
        String key = extractKey(m);
        keys.add(key);
        ByteBuf payload = extractPayload(m);
        byte[] bytes = new byte[payload.readableBytes()];
        payload.readBytes(bytes);
        Assert.assertEquals(bytes, expected.remove(key), "Compacted version should match expected version");
        m.close();
    }
    Assert.assertTrue(expected.isEmpty(), "All expected keys should have been found");
    return keys;
}

Example 9 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class RawMessageSerDeserTest method testSerializationAndDeserialization.

@Test
public void testSerializationAndDeserialization() throws Exception {
    int payload = 0xbeefcafe;
    ByteBuf headersAndPayload = Unpooled.buffer(4);
    headersAndPayload.writeInt(payload);
    MessageIdData id = MessageIdData.newBuilder().setLedgerId(0xf00).setEntryId(0xbaa).setPartition(10).setBatchIndex(20).build();
    @Cleanup RawMessage m = new RawMessageImpl(id, headersAndPayload);
    ByteBuf serialized = m.serialize();
    byte[] bytes = new byte[serialized.readableBytes()];
    serialized.readBytes(bytes);
    RawMessage m2 = RawMessageImpl.deserializeFrom(Unpooled.wrappedBuffer(bytes));
    Assert.assertEquals(m2.getMessageIdData().getLedgerId(), m.getMessageIdData().getLedgerId());
    Assert.assertEquals(m2.getMessageIdData().getEntryId(), m.getMessageIdData().getEntryId());
    Assert.assertEquals(m2.getMessageIdData().getPartition(), m.getMessageIdData().getPartition());
    Assert.assertEquals(m2.getMessageIdData().getBatchIndex(), m.getMessageIdData().getBatchIndex());
    Assert.assertEquals(m2.getHeadersAndPayload(), m.getHeadersAndPayload());
}

Example 10 with RawMessage

use of org.apache.pulsar.client.api.RawMessage in project incubator-pulsar by apache.

the class RawReaderTest method testSeekToMiddle.

@Test
public void testSeekToMiddle() throws Exception {
    int numKeys = 10;
    String topic = "persistent://my-property/use/my-ns/my-raw-topic";
    publishMessages(topic, numKeys);
    Set<String> readKeys = new HashSet<>();
    RawReader reader = RawReader.create(pulsarClient, topic, subscription).get();
    int i = 0;
    MessageId seekTo = null;
    MessageId lastMessageId = reader.getLastMessageIdAsync().get();
    while (true) {
        try (RawMessage m = reader.readNextAsync().get()) {
            i++;
            if (i > numKeys / 2) {
                if (seekTo == null) {
                    seekTo = m.getMessageId();
                }
                readKeys.add(extractKey(m));
            }
            if (lastMessageId.compareTo(m.getMessageId()) == 0) {
                break;
            }
        }
    }
    Assert.assertEquals(readKeys.size(), numKeys / 2);
    // seek to middle, read all keys again,
    // assert that we read all keys we had read previously
    reader.seekAsync(seekTo).get();
    while (true) {
        // should break out with TimeoutException
        try (RawMessage m = reader.readNextAsync().get()) {
            Assert.assertTrue(readKeys.remove(extractKey(m)));
            if (lastMessageId.compareTo(m.getMessageId()) == 0) {
                break;
            }
        }
    }
    Assert.assertTrue(readKeys.isEmpty());
}