use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by apache.
the class ParserProxyHandler method channelRead.
public void channelRead(ChannelHandlerContext ctx, Object msg) {
TopicName topicName;
List<RawMessage> messages = new ArrayList<>();
ByteBuf buffer = (ByteBuf) (msg);
try {
buffer.markReaderIndex();
buffer.markWriterIndex();
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
switch(cmd.getType()) {
case PRODUCER:
ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
break;
case SEND:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
MutableLong msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
case SUBSCRIBE:
ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
break;
case MESSAGE:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + DirectProxyHandler.inboundOutboundChannelMap.get(ctx.channel().id())));
msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
default:
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
} catch (Exception e) {
log.error("channelRead error ", e);
} finally {
buffer.resetReaderIndex();
buffer.resetWriterIndex();
// add totalSize to buffer Head
ByteBuf totalSizeBuf = Unpooled.buffer(4);
totalSizeBuf.writeInt(buffer.readableBytes());
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
compBuf.addComponents(totalSizeBuf, buffer);
compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
// Release mssages
messages.forEach(RawMessage::release);
// next handler
ctx.fireChannelRead(compBuf);
}
}
use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by apache.
the class ProxyStatsTest method testTopicStats.
/**
* Validate proxy topic stats api
*
* @throws Exception
*/
@Test
public void testTopicStats() throws Exception {
proxyService.setProxyLogLevel(2);
final String topicName = "persistent://sample/test/local/topic-stats";
final String topicName2 = "persistent://sample/test/local/topic-stats-2";
@Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false).producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false).producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
// Create a consumer directly attached to broker
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub").subscribe();
int totalMessages = 10;
for (int i = 0; i < totalMessages; i++) {
producer1.send("test".getBytes());
producer2.send("test".getBytes());
}
for (int i = 0; i < totalMessages; i++) {
Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
requireNonNull(msg);
consumer.acknowledge(msg);
msg = consumer2.receive(1, TimeUnit.SECONDS);
}
Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request().get();
Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
String response = r.readEntity(String.class).trim();
Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
}.getType());
assertNotNull(topicStats.get(topicName));
consumer.close();
consumer2.close();
}
use of org.apache.pulsar.proxy.stats.TopicStats in project pulsar by yahoo.
the class ParserProxyHandler method channelRead.
public void channelRead(ChannelHandlerContext ctx, Object msg) {
TopicName topicName;
List<RawMessage> messages = new ArrayList<>();
ByteBuf buffer = (ByteBuf) (msg);
try {
buffer.markReaderIndex();
buffer.markWriterIndex();
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
switch(cmd.getType()) {
case PRODUCER:
ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
break;
case SEND:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
MutableLong msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
case SUBSCRIBE:
ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
break;
case MESSAGE:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
default:
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
} catch (Exception e) {
log.error("channelRead error ", e);
} finally {
buffer.resetReaderIndex();
buffer.resetWriterIndex();
// add totalSize to buffer Head
ByteBuf totalSizeBuf = Unpooled.buffer(4);
totalSizeBuf.writeInt(buffer.readableBytes());
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
compBuf.addComponents(totalSizeBuf, buffer);
compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
// Release mssages
messages.forEach(RawMessage::release);
// next handler
ctx.fireChannelRead(compBuf);
}
}
use of org.apache.pulsar.proxy.stats.TopicStats in project incubator-pulsar by apache.
the class ProxyStatsTest method testTopicStats.
/**
* Validate proxy topic stats api
*
* @throws Exception
*/
@Test
public void testTopicStats() throws Exception {
proxyService.setProxyLogLevel(2);
final String topicName = "persistent://sample/test/local/topic-stats";
final String topicName2 = "persistent://sample/test/local/topic-stats-2";
@Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(proxyService.getServiceUrl()).build();
Producer<byte[]> producer1 = client.newProducer(Schema.BYTES).topic(topicName).enableBatching(false).producerName("producer1").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
Producer<byte[]> producer2 = client.newProducer(Schema.BYTES).topic(topicName2).enableBatching(false).producerName("producer2").messageRoutingMode(MessageRoutingMode.SinglePartition).create();
// Create a consumer directly attached to broker
Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName2).subscriptionName("my-sub").subscribe();
int totalMessages = 10;
for (int i = 0; i < totalMessages; i++) {
producer1.send("test".getBytes());
producer2.send("test".getBytes());
}
for (int i = 0; i < totalMessages; i++) {
Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
requireNonNull(msg);
consumer.acknowledge(msg);
msg = consumer2.receive(1, TimeUnit.SECONDS);
}
Client httpClient = ClientBuilder.newClient(new ClientConfig().register(LoggingFeature.class));
Response r = httpClient.target(proxyWebServer.getServiceUri()).path("/proxy-stats/topics").request().get();
Assert.assertEquals(r.getStatus(), Response.Status.OK.getStatusCode());
String response = r.readEntity(String.class).trim();
Map<String, TopicStats> topicStats = new Gson().fromJson(response, new TypeToken<Map<String, TopicStats>>() {
}.getType());
assertNotNull(topicStats.get(topicName));
consumer.close();
consumer2.close();
}
use of org.apache.pulsar.proxy.stats.TopicStats in project incubator-pulsar by apache.
the class ParserProxyHandler method channelRead.
public void channelRead(ChannelHandlerContext ctx, Object msg) {
TopicName topicName;
List<RawMessage> messages = new ArrayList<>();
ByteBuf buffer = (ByteBuf) (msg);
try {
buffer.markReaderIndex();
buffer.markWriterIndex();
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
switch(cmd.getType()) {
case PRODUCER:
ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
break;
case SEND:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
MutableLong msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
case SUBSCRIBE:
ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
break;
case MESSAGE:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
default:
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
} catch (Exception e) {
log.error("channelRead error ", e);
} finally {
buffer.resetReaderIndex();
buffer.resetWriterIndex();
// add totalSize to buffer Head
ByteBuf totalSizeBuf = Unpooled.buffer(4);
totalSizeBuf.writeInt(buffer.readableBytes());
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
compBuf.addComponents(totalSizeBuf, buffer);
compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
// Release mssages
messages.forEach(RawMessage::release);
// next handler
ctx.fireChannelRead(compBuf);
}
}
Aggregations