Examples with RawMessage org.apache.pulsar.common.api.raw.RawMessage used on opensource projects

Example 11 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.

the class MessageParserTest method testParseMessages.

@Test(dataProvider = "batchingAndCompression")
public void testParseMessages(boolean batchEnabled, CompressionType compressionType) throws Exception {
    final String topic = "persistent://my-tenant/my-ns/message-parse-test-" + batchEnabled + "-" + compressionType;
    final TopicName topicName = TopicName.get(topic);
    final int n = 10;
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).compressionType(compressionType).enableBatching(batchEnabled).batchingMaxPublishDelay(10, TimeUnit.SECONDS).topic(topic).create();
    ManagedCursor cursor = ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get()).getManagedLedger().newNonDurableCursor(PositionImpl.EARLIEST);
    if (batchEnabled) {
        for (int i = 0; i < n - 1; i++) {
            producer.sendAsync("hello-" + i);
        }
        producer.send("hello-" + (n - 1));
    } else {
        for (int i = 0; i < n; i++) {
            producer.send("Pulsar-" + i);
        }
    }
    if (batchEnabled) {
        Entry entry = cursor.readEntriesOrWait(1).get(0);
        List<RawMessage> messages = Lists.newArrayList();
        ByteBuf headsAndPayload = entry.getDataBuffer();
        try {
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), headsAndPayload, messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
        } finally {
            entry.release();
        }
        assertEquals(messages.size(), 10);
        for (int i = 0; i < n; i++) {
            assertEquals(messages.get(i).getData(), Unpooled.wrappedBuffer(("hello-" + i).getBytes()));
        }
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
            assertEquals(headsAndPayload.refCnt(), 0);
        });
    } else {
        // Read through raw data
        assertEquals(cursor.getNumberOfEntriesInBacklog(false), n);
        List<Entry> entries = cursor.readEntriesOrWait(n);
        assertEquals(entries.size(), n);
        List<ByteBuf> headsAndPayloadList = Lists.newArrayList();
        List<RawMessage> messages = Lists.newArrayList();
        for (Entry entry : entries) {
            ByteBuf headsAndPayload = entry.getDataBuffer();
            headsAndPayloadList.add(headsAndPayload);
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), entry.getDataBuffer(), messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
            entry.release();
        }
        assertEquals(messages.size(), 10);
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        for (ByteBuf byteBuf : headsAndPayloadList) {
            Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
                assertEquals(byteBuf.refCnt(), 0);
            });
        }
    }
}

Example 12 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by apache.

the class ParserProxyHandler method logging.

private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List<RawMessage> messages) {
    if (messages != null) {
        // lag
        StringBuilder infoBuilder = new StringBuilder(info);
        for (RawMessage message : messages) {
            infoBuilder.append("[").append(System.currentTimeMillis() - message.getPublishTime()).append("] ").append(new String(ByteBufUtil.getBytes(message.getData()), StandardCharsets.UTF_8));
        }
        info = infoBuilder.toString();
    }
    // log conn format is like from source to target
    switch(this.connType) {
        case ParserProxyHandler.FRONTEND_CONN:
            log.info(ParserProxyHandler.FRONTEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.remoteAddress() + conn.localAddress() + "]", cmdtype, info);
            break;
        case ParserProxyHandler.BACKEND_CONN:
            log.info(ParserProxyHandler.BACKEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.localAddress() + conn.remoteAddress() + "]", cmdtype, info);
            break;
    }
}