use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopics method resetCursor.
@POST
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/resetcursor/{timestamp}")
@ApiOperation(value = "Reset subscription to message position closest to absolute timestamp (in ms).", notes = "There should not be any active consumers on the subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic does not exist"), @ApiResponse(code = 405, message = "Not supported for global topics"), @ApiResponse(code = 412, message = "Subscription has active consumers") })
public void resetCursor(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("timestamp") long timestamp, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
destination = decode(destination);
DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
if (partitionMetadata.partitions > 0) {
int numParts = partitionMetadata.partitions;
int numPartException = 0;
Exception partitionException = null;
try {
for (int i = 0; i < numParts; i++) {
pulsar().getAdminClient().persistentTopics().resetCursor(dn.getPartition(i).toString(), subName, timestamp);
}
} catch (PreconditionFailedException pfe) {
// throw the last exception if all partitions get this error
// any other exception on partition is reported back to user
++numPartException;
partitionException = pfe;
} catch (Exception e) {
log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), dn, subName, timestamp, e);
throw new RestException(e);
}
// report an error to user if unable to reset for all partitions
if (numPartException == numParts) {
log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), dn, subName, timestamp, partitionException);
throw new RestException(Status.PRECONDITION_FAILED, partitionException.getMessage());
} else if (numPartException > 0 && log.isDebugEnabled()) {
log.debug("[{}][{}] partial errors for reset cursor on subscription {} to time {} - ", clientAppId(), destination, subName, timestamp, partitionException);
}
} else {
validateAdminOperationOnDestination(dn, authoritative);
log.info("[{}][{}] received reset cursor on subscription {} to time {}", clientAppId(), destination, subName, timestamp);
PersistentTopic topic = getTopicReference(dn);
try {
PersistentSubscription sub = topic.getPersistentSubscription(subName);
checkNotNull(sub);
sub.resetCursor(timestamp).get();
log.info("[{}][{}] reset cursor on subscription {} to time {}", clientAppId(), dn, subName, timestamp);
} catch (Exception e) {
Throwable t = e.getCause();
log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), dn, subName, timestamp, e);
if (e instanceof NullPointerException) {
throw new RestException(Status.NOT_FOUND, "Subscription not found");
} else if (e instanceof NotAllowedException) {
throw new RestException(Status.METHOD_NOT_ALLOWED, e.getMessage());
} else if (t instanceof SubscriptionBusyException) {
throw new RestException(Status.PRECONDITION_FAILED, "Subscription has active connected consumers");
} else if (t instanceof SubscriptionInvalidCursorPosition) {
throw new RestException(Status.PRECONDITION_FAILED, "Unable to find position for timestamp specified -" + t.getMessage());
} else {
throw new RestException(e);
}
}
}
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class PersistentTopics method peekNthMessage.
@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
destination = decode(destination);
DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
if (partitionMetadata.partitions > 0) {
throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
}
validateAdminOperationOnDestination(dn, authoritative);
PersistentTopic topic = getTopicReference(dn);
PersistentReplicator repl = null;
PersistentSubscription sub = null;
Entry entry = null;
if (subName.startsWith(topic.replicatorPrefix)) {
repl = getReplicatorReference(subName, topic);
} else {
sub = getSubscriptionReference(subName, topic);
}
try {
if (subName.startsWith(topic.replicatorPrefix)) {
entry = repl.peekNthMessage(messagePosition).get();
} else {
entry = sub.peekNthMessage(messagePosition).get();
}
checkNotNull(entry);
PositionImpl pos = (PositionImpl) entry.getPosition();
ByteBuf metadataAndPayload = entry.getDataBuffer();
// moves the readerIndex to the payload
MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
ResponseBuilder responseBuilder = Response.ok();
responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
for (KeyValue keyValue : metadata.getPropertiesList()) {
responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
}
if (metadata.hasPublishTime()) {
responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
}
// Decode if needed
CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
// Copy into a heap buffer for output stream compatibility
ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
data.writeBytes(uncompressedPayload);
uncompressedPayload.release();
StreamingOutput stream = new StreamingOutput() {
@Override
public void write(OutputStream output) throws IOException, WebApplicationException {
output.write(data.array(), data.arrayOffset(), data.readableBytes());
data.release();
}
};
return responseBuilder.entity(stream).build();
} catch (NullPointerException npe) {
throw new RestException(Status.NOT_FOUND, "Message not found");
} catch (Exception exception) {
log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
throw new RestException(exception);
} finally {
if (entry != null) {
entry.release();
}
}
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class Namespaces method unsubscribe.
private void unsubscribe(NamespaceName nsName, String bundleRange, String subscription) {
try {
List<PersistentTopic> topicList = pulsar().getBrokerService().getAllTopicsFromNamespaceBundle(nsName.toString(), nsName.toString() + "/" + bundleRange);
List<CompletableFuture<Void>> futures = Lists.newArrayList();
if (subscription.startsWith(pulsar().getConfiguration().getReplicatorPrefix())) {
throw new RestException(Status.PRECONDITION_FAILED, "Cannot unsubscribe a replication cursor");
} else {
for (PersistentTopic topic : topicList) {
PersistentSubscription sub = topic.getPersistentSubscription(subscription);
if (sub != null) {
futures.add(sub.delete());
}
}
}
FutureUtil.waitForAll(futures).get();
} catch (RestException re) {
throw re;
} catch (Exception e) {
log.error("[{}] Failed to unsubscribe {} for namespace {}/{}", clientAppId(), subscription, nsName.toString(), bundleRange, e);
if (e.getCause() instanceof SubscriptionBusyException) {
throw new RestException(Status.PRECONDITION_FAILED, "Subscription has active connected consumers");
}
throw new RestException(e.getCause());
}
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class BookieClientStatsGenerator method generate.
private Map<String, Map<String, PendingBookieOpsStats>> generate() throws Exception {
if (pulsar.getBrokerService() != null && pulsar.getBrokerService().getTopics() != null) {
pulsar.getBrokerService().getTopics().forEach((name, topicFuture) -> {
PersistentTopic persistentTopic = (PersistentTopic) topicFuture.getNow(null);
if (persistentTopic != null) {
DestinationName destinationName = DestinationName.get(persistentTopic.getName());
put(destinationName, persistentTopic.getManagedLedger().getStats().getPendingBookieOpsStats());
}
});
}
return nsBookieClientStatsMap;
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentTopic in project pulsar by yahoo.
the class ResendRequestTest method testSharedSingleAckedNormalTopic.
@Test(timeOut = testTimeout)
public void testSharedSingleAckedNormalTopic() throws Exception {
String key = "testSharedSingleAckedNormalTopic";
final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
final String subscriptionName = "my-shared-subscription-" + key;
final String messagePredicate = "my-message-" + key + "-";
final int totalMessages = 10;
// 1. producer connect
Producer producer = pulsarClient.createProducer(topicName);
PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
assertNotNull(topicRef);
assertEquals(topicRef.getProducers().size(), 1);
// 2. Create consumer
ConsumerConfiguration conf = new ConsumerConfiguration();
conf.setReceiverQueueSize(totalMessages / 2);
conf.setSubscriptionType(SubscriptionType.Shared);
Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
// 3. Producer publishes messages
for (int i = 0; i < totalMessages; i++) {
String message = messagePredicate + i;
producer.send(message.getBytes());
log.info("Producer produced " + message);
}
// 4. Receive messages
int receivedConsumer1 = 0, receivedConsumer2 = 0;
Message message1 = consumer1.receive();
Message message2 = consumer2.receive();
do {
if (message1 != null) {
log.info("Consumer 1 Received: " + new String(message1.getData()));
receivedConsumer1 += 1;
}
if (message2 != null) {
log.info("Consumer 2 Received: " + new String(message2.getData()));
receivedConsumer2 += 1;
}
message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
} while (message1 != null || message2 != null);
log.info("Consumer 1 receives = " + receivedConsumer1);
log.info("Consumer 2 receives = " + receivedConsumer2);
log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
// 5. Send a resend request from Consumer 1
log.info("Consumer 1 sent a resend request");
consumer1.redeliverUnacknowledgedMessages();
// 6. Consumer 1's unAcked messages should be sent to Consumer 1 or 2
int receivedMessagesAfterRedelivery = 0;
receivedConsumer1 = 0;
message1 = consumer1.receive(100, TimeUnit.MILLISECONDS);
message2 = consumer2.receive(100, TimeUnit.MILLISECONDS);
do {
if (message1 != null) {
log.info("Consumer 1 Received: " + new String(message1.getData()));
receivedConsumer1 += 1;
++receivedMessagesAfterRedelivery;
}
if (message2 != null) {
log.info("Consumer 2 Received: " + new String(message2.getData()));
receivedConsumer2 += 1;
++receivedMessagesAfterRedelivery;
}
message1 = consumer1.receive(200, TimeUnit.MILLISECONDS);
message2 = consumer2.receive(200, TimeUnit.MILLISECONDS);
} while (message1 != null || message2 != null);
log.info("Additional received = " + receivedMessagesAfterRedelivery);
assertTrue(receivedMessagesAfterRedelivery > 0);
assertEquals(receivedConsumer1 + receivedConsumer2, totalMessages);
}
Aggregations