use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.
the class ParserProxyHandler method logging.
private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List<RawMessage> messages) {
if (messages != null) {
// lag
StringBuilder infoBuilder = new StringBuilder(info);
for (RawMessage message : messages) {
infoBuilder.append("[").append(System.currentTimeMillis() - message.getPublishTime()).append("] ").append(new String(ByteBufUtil.getBytes(message.getData()), StandardCharsets.UTF_8));
}
info = infoBuilder.toString();
}
// log conn format is like from source to target
switch(this.connType) {
case ParserProxyHandler.FRONTEND_CONN:
log.info(ParserProxyHandler.FRONTEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.remoteAddress() + conn.localAddress() + "]", cmdtype, info);
break;
case ParserProxyHandler.BACKEND_CONN:
log.info(ParserProxyHandler.BACKEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.localAddress() + conn.remoteAddress() + "]", cmdtype, info);
break;
}
}
use of org.apache.pulsar.common.api.raw.RawMessage in project pulsar by yahoo.
the class ParserProxyHandler method channelRead.
public void channelRead(ChannelHandlerContext ctx, Object msg) {
TopicName topicName;
List<RawMessage> messages = new ArrayList<>();
ByteBuf buffer = (ByteBuf) (msg);
try {
buffer.markReaderIndex();
buffer.markWriterIndex();
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
switch(cmd.getType()) {
case PRODUCER:
ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
break;
case SEND:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
MutableLong msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
case SUBSCRIBE:
ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
break;
case MESSAGE:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
default:
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
} catch (Exception e) {
log.error("channelRead error ", e);
} finally {
buffer.resetReaderIndex();
buffer.resetWriterIndex();
// add totalSize to buffer Head
ByteBuf totalSizeBuf = Unpooled.buffer(4);
totalSizeBuf.writeInt(buffer.readableBytes());
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
compBuf.addComponents(totalSizeBuf, buffer);
compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
// Release mssages
messages.forEach(RawMessage::release);
// next handler
ctx.fireChannelRead(compBuf);
}
}
use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.
the class TestPulsarRecordCursor method testGetSchemaInfo.
@Test
public void testGetSchemaInfo() throws Exception {
String topic = "get-schema-test";
PulsarSplit pulsarSplit = Mockito.mock(PulsarSplit.class);
Mockito.when(pulsarSplit.getTableName()).thenReturn(TopicName.get(topic).getLocalName());
Mockito.when(pulsarSplit.getSchemaName()).thenReturn("public/default");
PulsarAdmin pulsarAdmin = Mockito.mock(PulsarAdmin.class);
Schemas schemas = Mockito.mock(Schemas.class);
Mockito.when(pulsarAdmin.schemas()).thenReturn(schemas);
PulsarConnectorConfig connectorConfig = spy(PulsarConnectorConfig.class);
Mockito.when(connectorConfig.getPulsarAdmin()).thenReturn(pulsarAdmin);
PulsarRecordCursor pulsarRecordCursor = spy(new PulsarRecordCursor(new ArrayList<>(), pulsarSplit, connectorConfig, Mockito.mock(ManagedLedgerFactory.class), new ManagedLedgerConfig(), null, null));
Class<PulsarRecordCursor> clazz = PulsarRecordCursor.class;
Method getSchemaInfo = clazz.getDeclaredMethod("getSchemaInfo", PulsarSplit.class);
getSchemaInfo.setAccessible(true);
Field currentMessage = clazz.getDeclaredField("currentMessage");
currentMessage.setAccessible(true);
RawMessage rawMessage = Mockito.mock(RawMessage.class);
currentMessage.set(pulsarRecordCursor, rawMessage);
// If the schemaType of pulsarSplit is NONE or BYTES, using bytes schema
Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.NONE);
SchemaInfo schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(SchemaType.BYTES, schemaInfo.getType());
Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.BYTES);
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(SchemaType.BYTES, schemaInfo.getType());
Mockito.when(pulsarSplit.getSchemaName()).thenReturn(Schema.BYTEBUFFER.getSchemaInfo().getName());
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(SchemaType.BYTES, schemaInfo.getType());
// If the schemaVersion of the message is not null, try to get the schema.
Mockito.when(pulsarSplit.getSchemaType()).thenReturn(SchemaType.AVRO);
Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(0).bytes());
Mockito.when(schemas.getSchemaInfo(anyString(), eq(0L))).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(SchemaType.AVRO, schemaInfo.getType());
String schemaTopic = "persistent://public/default/" + topic;
// If the schemaVersion of the message is null and the schema of pulsarSplit is null, throw runtime exception.
Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(null);
Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
try {
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
fail("The message schema version is null and the latest schema is null, should fail.");
} catch (InvocationTargetException e) {
assertTrue(e.getCause() instanceof RuntimeException);
assertTrue(e.getCause().getMessage().contains("schema of the table " + topic + " is null"));
}
// If the schemaVersion of the message is null, try to get the latest schema.
Mockito.when(rawMessage.getSchemaVersion()).thenReturn(null);
Mockito.when(pulsarSplit.getSchemaInfo()).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
// If the specific version schema is null, throw runtime exception.
Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(1L).bytes());
Mockito.when(schemas.getSchemaInfo(schemaTopic, 1)).thenReturn(null);
try {
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
fail("The specific version " + 1 + " schema is null, should fail.");
} catch (InvocationTargetException e) {
String schemaVersion = BytesSchemaVersion.of(new LongSchemaVersion(1L).bytes()).toString();
assertTrue(e.getCause() instanceof RuntimeException);
assertTrue(e.getCause().getMessage().contains("schema of the topic " + schemaTopic + " is null"));
}
// Get the specific version schema.
Mockito.when(rawMessage.getSchemaVersion()).thenReturn(new LongSchemaVersion(2L).bytes());
Mockito.when(schemas.getSchemaInfo(schemaTopic, 2)).thenReturn(Schema.AVRO(Foo.class).getSchemaInfo());
schemaInfo = (SchemaInfo) getSchemaInfo.invoke(pulsarRecordCursor, pulsarSplit);
assertEquals(Schema.AVRO(Foo.class).getSchemaInfo(), schemaInfo);
}
use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.
the class ParserProxyHandler method logging.
private void logging(Channel conn, BaseCommand.Type cmdtype, String info, List<RawMessage> messages) {
if (messages != null) {
// lag
StringBuilder infoBuilder = new StringBuilder(info);
for (RawMessage message : messages) {
infoBuilder.append("[").append(System.currentTimeMillis() - message.getPublishTime()).append("] ").append(new String(ByteBufUtil.getBytes(message.getData()), StandardCharsets.UTF_8));
}
info = infoBuilder.toString();
}
// log conn format is like from source to target
switch(this.connType) {
case ParserProxyHandler.FRONTEND_CONN:
log.info(ParserProxyHandler.FRONTEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.remoteAddress() + conn.localAddress() + "]", cmdtype, info);
break;
case ParserProxyHandler.BACKEND_CONN:
log.info(ParserProxyHandler.BACKEND_CONN + ":{} cmd:{} msg:{}", "[" + conn.localAddress() + conn.remoteAddress() + "]", cmdtype, info);
break;
}
}
use of org.apache.pulsar.common.api.raw.RawMessage in project incubator-pulsar by apache.
the class ParserProxyHandler method channelRead.
public void channelRead(ChannelHandlerContext ctx, Object msg) {
TopicName topicName;
List<RawMessage> messages = new ArrayList<>();
ByteBuf buffer = (ByteBuf) (msg);
try {
buffer.markReaderIndex();
buffer.markWriterIndex();
int cmdSize = (int) buffer.readUnsignedInt();
cmd.parseFrom(buffer, cmdSize);
switch(cmd.getType()) {
case PRODUCER:
ParserProxyHandler.producerHashMap.put(cmd.getProducer().getProducerId() + "," + ctx.channel().id(), cmd.getProducer().getTopic());
logging(ctx.channel(), cmd.getType(), "{producer:" + cmd.getProducer().getProducerName() + ",topic:" + cmd.getProducer().getTopic() + "}", null);
break;
case SEND:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.producerHashMap.get(cmd.getSend().getProducerId() + "," + ctx.channel().id()));
MutableLong msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
TopicStats topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgInRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
case SUBSCRIBE:
ParserProxyHandler.consumerHashMap.put(cmd.getSubscribe().getConsumerId() + "," + ctx.channel().id(), cmd.getSubscribe().getTopic());
logging(ctx.channel(), cmd.getType(), "{consumer:" + cmd.getSubscribe().getConsumerName() + ",topic:" + cmd.getSubscribe().getTopic() + "}", null);
break;
case MESSAGE:
if (service.getProxyLogLevel() != 2) {
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
topicName = TopicName.get(ParserProxyHandler.consumerHashMap.get(cmd.getMessage().getConsumerId() + "," + peerChannelId));
msgBytes = new MutableLong(0);
MessageParser.parseMessage(topicName, -1L, -1L, buffer, (message) -> {
messages.add(message);
msgBytes.add(message.getData().readableBytes());
}, maxMessageSize);
// update topic stats
topicStats = this.service.getTopicStats().computeIfAbsent(topicName.toString(), topic -> new TopicStats());
topicStats.getMsgOutRate().recordMultipleEvents(messages.size(), msgBytes.longValue());
logging(ctx.channel(), cmd.getType(), "", messages);
break;
default:
logging(ctx.channel(), cmd.getType(), "", null);
break;
}
} catch (Exception e) {
log.error("channelRead error ", e);
} finally {
buffer.resetReaderIndex();
buffer.resetWriterIndex();
// add totalSize to buffer Head
ByteBuf totalSizeBuf = Unpooled.buffer(4);
totalSizeBuf.writeInt(buffer.readableBytes());
CompositeByteBuf compBuf = Unpooled.compositeBuffer();
compBuf.addComponents(totalSizeBuf, buffer);
compBuf.writerIndex(totalSizeBuf.capacity() + buffer.capacity());
// Release mssages
messages.forEach(RawMessage::release);
// next handler
ctx.fireChannelRead(compBuf);
}
}
Aggregations