Search in sources :

Example 6 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.

the class ParserProxyHandler method logging.

private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List<RawMessage> messages) {
    if (messages != null) {
        // lag
        StringBuilder infoBuilder = new StringBuilder(info);
        for (RawMessage message : messages) {
            infoBuilder.append("[").append(System.currentTimeMillis() - message.getPublishTime()).append("] ").append(new String(ByteBufUtil.getBytes(message.getData()), StandardCharsets.UTF_8));
        }
        info = infoBuilder.toString();
    }
    // log conn format is like from source to target
    switch(this.connType) {
        case ParserProxyHandler.FRONTEND_CONN:
            log.info(ParserProxyHandler.FRONTEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.remoteAddress() + conn.localAddress() + "]", cmdtype, info);
            break;
        case ParserProxyHandler.BACKEND_CONN:
            log.info(ParserProxyHandler.BACKEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.localAddress() + conn.remoteAddress() + "]", cmdtype, info);
            break;
    }
}
Also used : RawMessage(org.apache.pulsar.common.api.raw.RawMessage)

Example 7 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) ChannelId(io.netty.channel.ChannelId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MessageParser(org.apache.pulsar.common.api.raw.MessageParser) LoggerFactory(org.slf4j.LoggerFactory) StandardCharsets(java.nio.charset.StandardCharsets) Unpooled(io.netty.buffer.Unpooled) ArrayList(java.util.ArrayList) Channel(io.netty.channel.Channel) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBufUtil(io.netty.buffer.ByteBufUtil) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ArrayList(java.util.ArrayList) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 8 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.

the class TestPulsarRecordCursor method testGetSchemaInfo.

@Test
public void testGetSchemaInfo() throws Exception {
    String topic = "get-schema-test";
    PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class);
    Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default");
    PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class);
    Schemas schemas = Mockito.mock(Schemas.class);
    Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas);
    PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class);
    Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin);
    PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), new ManagedLedgerConfig(), null, null));
    Class<PulsarRecordCursor> clazz = PulsarRecordCursor.class;
    Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class);
    getSchemaInfo.setAccessible(true);
    Field currentMessage = clazz.getDeclaredField("currentMessage");
    currentMessage.setAccessible(true);
    RawMessage rawMessage = Mockito.mock(RawMessage.class);
    currentMessage.set(pulsarRecordCursor, rawMessage);
    // If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE);
    SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES);
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.BYTES, schemaInfo.getType());
    // If the schemaVersion of the message is not null, try to get the schema.
    Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
    Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(SchemaType.AVRO, schemaInfo.getType());
    String schemaTopic = "persistent://public/default/" + topic;
    // If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception.
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null);
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The message schema version is null and the latest schema is null, should fail.");
    } catch (InvocationTargetException e) {
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
    }
    // If the schemaVersion of the message is null, try to get the latest schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
    Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
    // If the specific version schema is null, throw runtime exception.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
    try {
        schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
        fail("The specific version " + 1 + " schema is null, should fail.");
    } catch (InvocationTargetException e) {
        String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
        assertTrue(e.getCause() instanceof RuntimeException);
        assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
    }
    // Get the specific version schema.
    Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
    Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
    schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
    assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
}
Also used : PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) ArrayList(java.util.ArrayList) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) Method(java.lang.reflect.Method) Schemas(org.apache.pulsar.client.admin.Schemas) InvocationTargetException(java.lang.reflect.InvocationTargetException) Field(java.lang.reflect.Field) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) SchemaInfo(org.apache.pulsar.common.schema.SchemaInfo) LongSchemaVersion(org.apache.pulsar.common.schema.LongSchemaVersion) Test(org.testng.annotations.Test)

Example 9 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.

the class ParserProxyHandler method logging.

private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List<RawMessage> messages) {
    if (messages != null) {
        // lag
        StringBuilder infoBuilder = new StringBuilder(info);
        for (RawMessage message : messages) {
            infoBuilder.append("[").append(System.currentTimeMillis() - message.getPublishTime()).append("] ").append(new String(ByteBufUtil.getBytes(message.getData()), StandardCharsets.UTF_8));
        }
        info = infoBuilder.toString();
    }
    // log conn format is like from source to target
    switch(this.connType) {
        case ParserProxyHandler.FRONTEND_CONN:
            log.info(ParserProxyHandler.FRONTEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.remoteAddress() + conn.localAddress() + "]", cmdtype, info);
            break;
        case ParserProxyHandler.BACKEND_CONN:
            log.info(ParserProxyHandler.BACKEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.localAddress() + conn.remoteAddress() + "]", cmdtype, info);
            break;
    }
}
Also used : RawMessage(org.apache.pulsar.common.api.raw.RawMessage)

Example 10 with RawMessage

use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) ChannelId(io.netty.channel.ChannelId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MessageParser(org.apache.pulsar.common.api.raw.MessageParser) LoggerFactory(org.slf4j.LoggerFactory) StandardCharsets(java.nio.charset.StandardCharsets) Unpooled(io.netty.buffer.Unpooled) ArrayList(java.util.ArrayList) Channel(io.netty.channel.Channel) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBufUtil(io.netty.buffer.ByteBufUtil) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ArrayList(java.util.ArrayList) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) TopicName(org.apache.pulsar.common.naming.TopicName)

Aggregations

RawMessage (org.apache.pulsar.common.api.raw.RawMessage)12 ByteBuf (io.netty.buffer.ByteBuf)6 ArrayList (java.util.ArrayList)6 TopicName (org.apache.pulsar.common.naming.TopicName)6 Test (org.testng.annotations.Test)6 ByteBufUtil (io.netty.buffer.ByteBufUtil)3 CompositeByteBuf (io.netty.buffer.CompositeByteBuf)3 Unpooled (io.netty.buffer.Unpooled)3 Channel (io.netty.channel.Channel)3 ChannelHandlerContext (io.netty.channel.ChannelHandlerContext)3 ChannelInboundHandlerAdapter (io.netty.channel.ChannelInboundHandlerAdapter)3 Field (java.lang.reflect.Field)3 InvocationTargetException (java.lang.reflect.InvocationTargetException)3 Method (java.lang.reflect.Method)3 StandardCharsets (java.nio.charset.StandardCharsets)3 List (java.util.List)3 Map (java.util.Map)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 Cleanup (lombok.Cleanup)3 Entry (org.apache.bookkeeper.mledger.Entry)3