Examples with RawMessage org.apache.pulsar.common.api.raw.RawMessage used on opensource projects

Example 1 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by apache.

the class MessageParserTest method testParseMessages.

@Test(dataProvider = "batchingAndCompression")
public void testParseMessages(boolean batchEnabled, CompressionType compressionType) throws Exception {
    final String topic = "persistent://my-tenant/my-ns/message-parse-test-" + batchEnabled + "-" + compressionType;
    final TopicName topicName = TopicName.get(topic);
    final int n = 10;
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).compressionType(compressionType).enableBatching(batchEnabled).batchingMaxPublishDelay(10, TimeUnit.SECONDS).topic(topic).create();
    ManagedCursor cursor = ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get()).getManagedLedger().newNonDurableCursor(PositionImpl.EARLIEST);
    if (batchEnabled) {
        for (int i = 0; i < n - 1; i++) {
            producer.sendAsync("hello-" + i);
        }
        producer.send("hello-" + (n - 1));
    } else {
        for (int i = 0; i < n; i++) {
            producer.send("Pulsar-" + i);
        }
    }
    if (batchEnabled) {
        Entry entry = cursor.readEntriesOrWait(1).get(0);
        List<RawMessage> messages = Lists.newArrayList();
        ByteBuf headsAndPayload = entry.getDataBuffer();
        try {
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), headsAndPayload, messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
        } finally {
            entry.release();
        }
        assertEquals(messages.size(), 10);
        for (int i = 0; i < n; i++) {
            assertEquals(messages.get(i).getData(), Unpooled.wrappedBuffer(("hello-" + i).getBytes()));
        }
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
            assertEquals(headsAndPayload.refCnt(), 0);
        });
    } else {
        // Read through raw data
        assertEquals(cursor.getNumberOfEntriesInBacklog(false), n);
        List<Entry> entries = cursor.readEntriesOrWait(n);
        assertEquals(entries.size(), n);
        List<ByteBuf> headsAndPayloadList = Lists.newArrayList();
        List<RawMessage> messages = Lists.newArrayList();
        for (Entry entry : entries) {
            ByteBuf headsAndPayload = entry.getDataBuffer();
            headsAndPayloadList.add(headsAndPayload);
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), entry.getDataBuffer(), messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
            entry.release();
        }
        assertEquals(messages.size(), 10);
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        for (ByteBuf byteBuf : headsAndPayloadList) {
            Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
                assertEquals(byteBuf.refCnt(), 0);
            });
        }
    }
}

Example 2 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by apache.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + DirectProxyHandler.inboundOutboundChannelMap.get(ctx.channel().id())));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}

Example 3 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by apache.

the class TestPulsarRecordCursor method testGetSchemaInfo.

@Test
public void testGetSchemaInfo() throws Exception {
    String topic = "get-schema-test";
    PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class);
    Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default");
    PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class);
    Schemas schemas = Mockito.mock(Schemas.class);
    Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas);
    PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class);
    Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), new ManagedLedgerConfig(), null, null));
    Class<PulsarRecordCursor> clazz = PulsarRecordCursor.class;
    Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class);
    getSchemaInfo.setAccessible(true);
    Field currentMessage = clazz.getDeclaredField("currentMessage");
    currentMessage.setAccessible(true);
    RawMessage rawMessage = Mockito.mock(RawMessage.class);
    currentMessage.set(pulsarRecordCursor, rawMessage);
    // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE);
    SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES);
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    // If the schemaVersion of the message is not null, try to get the schema.
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
    Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.AVRO, schemaInfo.getType());
    String schemaTopic = "persistent://public/default/" + topic;
    // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception.
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The message schema version is null and the latest schema is null, should fail.");
    } catch (InvocationTargetException e) {
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
    }
    // If the schemaVersion of the message is null, try to get the latest schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
    // If the specific version schema is null, throw runtime exception.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The specific version " + 1 + " schema is null, should fail.");
    } catch (InvocationTargetException e) {
        String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
    }
    // Get the specific version schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
}

Example 4 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.

the class MessageParserTest method testParseMessages.

@Test(dataProvider = "batchingAndCompression")
public void testParseMessages(boolean batchEnabled, CompressionType compressionType) throws Exception {
    final String topic = "persistent://my-tenant/my-ns/message-parse-test-" + batchEnabled + "-" + compressionType;
    final TopicName topicName = TopicName.get(topic);
    final int n = 10;
    @Cleanup Producer<String> producer = pulsarClient.newProducer(Schema.STRING).compressionType(compressionType).enableBatching(batchEnabled).batchingMaxPublishDelay(10, TimeUnit.SECONDS).topic(topic).create();
    ManagedCursor cursor = ((PersistentTopic) pulsar.getBrokerService().getTopicReference(topic).get()).getManagedLedger().newNonDurableCursor(PositionImpl.EARLIEST);
    if (batchEnabled) {
        for (int i = 0; i < n - 1; i++) {
            producer.sendAsync("hello-" + i);
        }
        producer.send("hello-" + (n - 1));
    } else {
        for (int i = 0; i < n; i++) {
            producer.send("Pulsar-" + i);
        }
    }
    if (batchEnabled) {
        Entry entry = cursor.readEntriesOrWait(1).get(0);
        List<RawMessage> messages = Lists.newArrayList();
        ByteBuf headsAndPayload = entry.getDataBuffer();
        try {
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), headsAndPayload, messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
        } finally {
            entry.release();
        }
        assertEquals(messages.size(), 10);
        for (int i = 0; i < n; i++) {
            assertEquals(messages.get(i).getData(), Unpooled.wrappedBuffer(("hello-" + i).getBytes()));
        }
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
            assertEquals(headsAndPayload.refCnt(), 0);
        });
    } else {
        // Read through raw data
        assertEquals(cursor.getNumberOfEntriesInBacklog(false), n);
        List<Entry> entries = cursor.readEntriesOrWait(n);
        assertEquals(entries.size(), n);
        List<ByteBuf> headsAndPayloadList = Lists.newArrayList();
        List<RawMessage> messages = Lists.newArrayList();
        for (Entry entry : entries) {
            ByteBuf headsAndPayload = entry.getDataBuffer();
            headsAndPayloadList.add(headsAndPayload);
            MessageParser.parseMessage(topicName, entry.getLedgerId(), entry.getEntryId(), entry.getDataBuffer(), messages::add, Commands.DEFAULT_MAX_MESSAGE_SIZE);
            entry.release();
        }
        assertEquals(messages.size(), 10);
        messages.forEach(msg -> {
            msg.getSchemaVersion();
            msg.release();
        });
        for (ByteBuf byteBuf : headsAndPayloadList) {
            Awaitility.await().atMost(3, TimeUnit.SECONDS).untilAsserted(() -> {
                assertEquals(byteBuf.refCnt(), 0);
            });
        }
    }
}

Example 5 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.

the class TestPulsarRecordCursor method testGetSchemaInfo.

@Test
public void testGetSchemaInfo() throws Exception {
    String topic = "get-schema-test";
    PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class);
    Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default");
    PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class);
    Schemas schemas = Mockito.mock(Schemas.class);
    Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas);
    PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class);
    Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), new ManagedLedgerConfig(), null, null));
    Class<PulsarRecordCursor> clazz = PulsarRecordCursor.class;
    Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class);
    getSchemaInfo.setAccessible(true);
    Field currentMessage = clazz.getDeclaredField("currentMessage");
    currentMessage.setAccessible(true);
    RawMessage rawMessage = Mockito.mock(RawMessage.class);
    currentMessage.set(pulsarRecordCursor, rawMessage);
    // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE);
    SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES);
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    // If the schemaVersion of the message is not null, try to get the schema.
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
    Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.AVRO, schemaInfo.getType());
    String schemaTopic = "persistent://public/default/" + topic;
    // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception.
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The message schema version is null and the latest schema is null, should fail.");
    } catch (InvocationTargetException e) {
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
    }
    // If the schemaVersion of the message is null, try to get the latest schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
    // If the specific version schema is null, throw runtime exception.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The specific version " + 1 + " schema is null, should fail.");
    } catch (InvocationTargetException e) {
        String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
    }
    // Get the specific version schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
}