Search in sources :

Example 1 with TopicStats

use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by apache.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + DirectProxyHandler.inboundOutboundChannelMap.get(ctx.channel().id())));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MessageParser(org.apache.pulsar.common.api.raw.MessageParser) LoggerFactory(org.slf4j.LoggerFactory) StandardCharsets(java.nio.charset.StandardCharsets) Unpooled(io.netty.buffer.Unpooled) ArrayList(java.util.ArrayList) Channel(io.netty.channel.Channel) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBufUtil(io.netty.buffer.ByteBufUtil) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ArrayList(java.util.ArrayList) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 2 with TopicStats

use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by apache.

the class ProxyStatsTest method testTopicStats.

/**
 * Validate proxy topic stats api
 *
 * @throws Exception
 */
@Test
public void testTopicStats() throws Exception {
    proxyService.setProxyLogLevel(2);
    final String topicName = "persistent://sample/test/local/topic-stats";
    final String topicName2 = "persistent://sample/test/local/topic-stats-2";
    @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false).producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false).producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub").subscribe();
    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer1.send("test".getBytes());
        producer2.send("test".getBytes());
    }
    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        requireNonNull(msg);
        consumer.acknowledge(msg);
        msg = consumer2.receive(1, TimeUnit.SECONDS);
    }
    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request().get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
    }.getType());
    assertNotNull(topicStats.get(topicName));
    consumer.close();
    consumer2.close();
}
Also used : Gson(com.google.gson.Gson) Cleanup(lombok.Cleanup) Response(javax.ws.rs.core.Response) TypeToken(com.google.gson.reflect.TypeToken) LoggingFeature(org.glassfish.jersey.logging.LoggingFeature) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Client(javax.ws.rs.client.Client) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ClientConfig(org.glassfish.jersey.client.ClientConfig) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Example 3 with TopicStats

use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by yahoo.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) ChannelId(io.netty.channel.ChannelId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MessageParser(org.apache.pulsar.common.api.raw.MessageParser) LoggerFactory(org.slf4j.LoggerFactory) StandardCharsets(java.nio.charset.StandardCharsets) Unpooled(io.netty.buffer.Unpooled) ArrayList(java.util.ArrayList) Channel(io.netty.channel.Channel) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBufUtil(io.netty.buffer.ByteBufUtil) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ArrayList(java.util.ArrayList) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 4 with TopicStats

use of org.apache.pulsar.proxy.stats.TopicStats in project incubator-pulsar by apache.

the class ProxyStatsTest method testTopicStats.

/**
 * Validate proxy topic stats api
 *
 * @throws Exception
 */
@Test
public void testTopicStats() throws Exception {
    proxyService.setProxyLogLevel(2);
    final String topicName = "persistent://sample/test/local/topic-stats";
    final String topicName2 = "persistent://sample/test/local/topic-stats-2";
    @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
    Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false).producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false).producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    // Create a consumer directly attached to broker
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub").subscribe();
    int totalMessages = 10;
    for (int i = 0; i < totalMessages; i++) {
        producer1.send("test".getBytes());
        producer2.send("test".getBytes());
    }
    for (int i = 0; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        requireNonNull(msg);
        consumer.acknowledge(msg);
        msg = consumer2.receive(1, TimeUnit.SECONDS);
    }
    Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
    Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request().get();
    Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
    String response = r.readEntity(String.class).trim();
    Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
    }.getType());
    assertNotNull(topicStats.get(topicName));
    consumer.close();
    consumer2.close();
}
Also used : Gson(com.google.gson.Gson) Cleanup(lombok.Cleanup) Response(javax.ws.rs.core.Response) TypeToken(com.google.gson.reflect.TypeToken) LoggingFeature(org.glassfish.jersey.logging.LoggingFeature) PulsarClient(org.apache.pulsar.client.api.PulsarClient) Client(javax.ws.rs.client.Client) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ClientConfig(org.glassfish.jersey.client.ClientConfig) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Example 5 with TopicStats

use of org.apache.pulsar.proxy.stats.TopicStats in project incubator-pulsar by apache.

the class ParserProxyHandler method channelRead.

public void channelRead(ChannelHandlerContext ctx, Object msg) {
    TopicName topicName;
    List<RawMessage> messages = new ArrayList<>();
    ByteBuf buffer = (ByteBuf) (msg);
    try {
        buffer.markReaderIndex();
        buffer.markWriterIndex();
        int cmdSize = (int) buffer.readUnsignedInt();
        cmd.parseFrom(buffer, cmdSize);
        switch(cmd.getType()) {
            case PRODUCER:
                ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
                logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
                break;
            case SEND:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
                MutableLong msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            case SUBSCRIBE:
                ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
                logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
                break;
            case MESSAGE:
                if (service.getProxyLogLevel() != 2) {
                    logging(ctx.channel(), cmd.getType(), "", null);
                    break;
                }
                topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
                msgBytes = new MutableLong(0);
                MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
                    messages.add(message);
                    msgBytes.add(message.getData().readableBytes());
                }, maxMessageSize);
                // update topic stats
                topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
                topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
                logging(ctx.channel(), cmd.getType(), "", messages);
                break;
            default:
                logging(ctx.channel(), cmd.getType(), "", null);
                break;
        }
    } catch (Exception e) {
        log.error("channelRead error ", e);
    } finally {
        buffer.resetReaderIndex();
        buffer.resetWriterIndex();
        // add totalSize to buffer Head
        ByteBuf totalSizeBuf = Unpooled.buffer(4);
        totalSizeBuf.writeInt(buffer.readableBytes());
        CompositeByteBuf compBuf = Unpooled.compositeBuffer();
        compBuf.addComponents(totalSizeBuf, buffer);
        compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
        // Release mssages
        messages.forEach(RawMessage::release);
        // next handler
        ctx.fireChannelRead(compBuf);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) Logger(org.slf4j.Logger) ChannelInboundHandlerAdapter(io.netty.channel.ChannelInboundHandlerAdapter) ChannelId(io.netty.channel.ChannelId) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MessageParser(org.apache.pulsar.common.api.raw.MessageParser) LoggerFactory(org.slf4j.LoggerFactory) StandardCharsets(java.nio.charset.StandardCharsets) Unpooled(io.netty.buffer.Unpooled) ArrayList(java.util.ArrayList) Channel(io.netty.channel.Channel) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBufUtil(io.netty.buffer.ByteBufUtil) List(java.util.List) ByteBuf(io.netty.buffer.ByteBuf) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) MutableLong(org.apache.commons.lang3.mutable.MutableLong) ArrayList(java.util.ArrayList) RawMessage(org.apache.pulsar.common.api.raw.RawMessage) CompositeByteBuf(io.netty.buffer.CompositeByteBuf) ByteBuf(io.netty.buffer.ByteBuf) TopicStats(org.apache.pulsar.proxy.stats.TopicStats) TopicName(org.apache.pulsar.common.naming.TopicName)

Aggregations

TopicStats (org.apache.pulsar.proxy.stats.TopicStats)6 Gson (com.google.gson.Gson)3 TypeToken (com.google.gson.reflect.TypeToken)3 ByteBuf (io.netty.buffer.ByteBuf)3 ByteBufUtil (io.netty.buffer.ByteBufUtil)3 CompositeByteBuf (io.netty.buffer.CompositeByteBuf)3 Unpooled (io.netty.buffer.Unpooled)3 Channel (io.netty.channel.Channel)3 ChannelHandlerContext (io.netty.channel.ChannelHandlerContext)3 ChannelInboundHandlerAdapter (io.netty.channel.ChannelInboundHandlerAdapter)3 StandardCharsets (java.nio.charset.StandardCharsets)3 ArrayList (java.util.ArrayList)3 List (java.util.List)3 Map (java.util.Map)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 Client (javax.ws.rs.client.Client)3 Response (javax.ws.rs.core.Response)3 Cleanup (lombok.Cleanup)3 MutableLong (org.apache.commons.lang3.mutable.MutableLong)3 MockedPulsarServiceBaseTest (org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)3