use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.
the class PersistentTopics method peekNthMessage.
@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
destination = decode(destination);
DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
if (partitionMetadata.partitions > 0) {
throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
}
validateAdminOperationOnDestination(dn, authoritative);
PersistentTopic topic = getTopicReference(dn);
PersistentReplicator repl = null;
PersistentSubscription sub = null;
Entry entry = null;
if (subName.startsWith(topic.replicatorPrefix)) {
repl = getReplicatorReference(subName, topic);
} else {
sub = getSubscriptionReference(subName, topic);
}
try {
if (subName.startsWith(topic.replicatorPrefix)) {
entry = repl.peekNthMessage(messagePosition).get();
} else {
entry = sub.peekNthMessage(messagePosition).get();
}
checkNotNull(entry);
PositionImpl pos = (PositionImpl) entry.getPosition();
ByteBuf metadataAndPayload = entry.getDataBuffer();
// moves the readerIndex to the payload
MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
ResponseBuilder responseBuilder = Response.ok();
responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
for (KeyValue keyValue : metadata.getPropertiesList()) {
responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
}
if (metadata.hasPublishTime()) {
responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
}
// Decode if needed
CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
// Copy into a heap buffer for output stream compatibility
ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
data.writeBytes(uncompressedPayload);
uncompressedPayload.release();
StreamingOutput stream = new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
output.write(data.array(), data.arrayOffset(), data.readableBytes());
data.release();
}
};
return responseBuilder.entity(stream).build();
} catch (NullPointerException npe) {
throw new RestException(Status.NOT_FOUND, "Message not found");
} catch (Exception exception) {
log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
throw new RestException(exception);
} finally {
if (entry != null) {
entry.release();
}
}
}
use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.
the class ServerCnxTest method testSendCommand.
@Test(timeOut = 30000)
public void testSendCommand() throws Exception {
resetChannel();
setChannelConnected();
ByteBuf clientCommand = Commands.newProducer(successTopicName, 1, /* producer id */
1, /* request id */
"prod-name");
channel.writeInbound(clientCommand);
assertTrue(getResponse() instanceof CommandProducerSuccess);
// test SEND success
MessageMetadata messageMetadata = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
ByteBuf data = Unpooled.buffer(1024);
clientCommand = Commands.newSend(1, 0, 1, ChecksumType.None, messageMetadata, data);
channel.writeInbound(Unpooled.copiedBuffer(clientCommand));
clientCommand.release();
assertTrue(getResponse() instanceof CommandSendReceipt);
channel.finish();
}
use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.
the class ManagedLedgerTest method getMessageWithMetadata.
public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length);
int msgMetadataSize = messageData.getSerializedSize();
int headersSize = 4 + msgMetadataSize;
ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
headers.writeInt(msgMetadataSize);
messageData.writeTo(outStream);
outStream.recycle();
return DoubleByteBuf.get(headers, payload);
}
use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.
the class ManagedLedgerImpl method checkBackloggedCursors.
@Override
public void checkBackloggedCursors() {
// activate caught up cursors
cursors.forEach(cursor -> {
if (cursor.getNumberOfEntries() < maxActiveCursorBacklogEntries) {
cursor.setActive();
}
});
// deactivate backlog cursors
Iterator<ManagedCursor> cursors = activeCursors.iterator();
while (cursors.hasNext()) {
ManagedCursor cursor = cursors.next();
long backlogEntries = cursor.getNumberOfEntries();
if (backlogEntries > maxActiveCursorBacklogEntries) {
PositionImpl readPosition = (PositionImpl) cursor.getReadPosition();
readPosition = isValidPosition(readPosition) ? readPosition : getNextValidPosition(readPosition);
if (readPosition == null) {
if (log.isDebugEnabled()) {
log.debug("[{}] Couldn't find valid read position [{}] {}", name, cursor.getName(), cursor.getReadPosition());
}
continue;
}
try {
asyncReadEntry(readPosition, new ReadEntryCallback() {
@Override
public void readEntryFailed(ManagedLedgerException e, Object ctx) {
log.warn("[{}] Failed while reading entries on [{}] {}", name, cursor.getName(), e.getMessage(), e);
}
@Override
public void readEntryComplete(Entry entry, Object ctx) {
MessageMetadata msgMetadata = null;
try {
msgMetadata = Commands.parseMessageMetadata(entry.getDataBuffer());
long msgTimeSincePublish = (System.currentTimeMillis() - msgMetadata.getPublishTime());
if (msgTimeSincePublish > maxMessageCacheRetentionTimeMillis) {
cursor.setInactive();
}
} finally {
if (msgMetadata != null) {
msgMetadata.recycle();
}
entry.release();
}
}
}, null);
} catch (Exception e) {
log.warn("[{}] Failed while reading entries from cache on [{}] {}", name, cursor.getName(), e.getMessage(), e);
}
}
}
}
use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.
the class PersistentTopicTest method testPublishMessageMLFailure.
@Test
public void testPublishMessageMLFailure() throws Exception {
final String successTopicName = "persistent://prop/use/ns-abc/successTopic";
final ManagedLedger ledgerMock = mock(ManagedLedger.class);
doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
MessageMetadata.Builder messageMetadata = MessageMetadata.newBuilder();
messageMetadata.setPublishTime(System.currentTimeMillis());
messageMetadata.setProducerName("prod-name");
messageMetadata.setSequenceId(1);
ByteBuf payload = Unpooled.wrappedBuffer("content".getBytes());
final CountDownLatch latch = new CountDownLatch(1);
// override asyncAddEntry callback to return error
doAnswer(new Answer<Object>() {
@Override
public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
((AddEntryCallback) invocationOnMock.getArguments()[1]).addFailed(new ManagedLedgerException("Managed ledger failure"), invocationOnMock.getArguments()[2]);
return null;
}
}).when(ledgerMock).asyncAddEntry(any(ByteBuf.class), any(AddEntryCallback.class), anyObject());
topic.publishMessage(payload, (exception, ledgerId, entryId) -> {
if (exception == null) {
fail("publish should have failed");
} else {
latch.countDown();
}
});
assertTrue(latch.await(1, TimeUnit.SECONDS));
}
Aggregations