use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.
the class PersistentTopicTest method testClosingReplicationProducerTwice.
@Test
public void testClosingReplicationProducerTwice() throws Exception {
final String globalTopicName = "persistent://prop/global/ns/testClosingReplicationProducerTwice";
String localCluster = "local";
String remoteCluster = "remote";
final ManagedLedger ledgerMock = mock(ManagedLedger.class);
doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
String remoteReplicatorName = topic.replicatorPrefix + "." + localCluster;
final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
PulsarClient client = spy(PulsarClient.create(brokerUrl.toString()));
PulsarClientImpl clientImpl = (PulsarClientImpl) client;
Field conf = PersistentReplicator.class.getDeclaredField("producerConfiguration");
conf.setAccessible(true);
ManagedCursor cursor = mock(ManagedCursorImpl.class);
doReturn(remoteCluster).when(cursor).getName();
brokerService.getReplicationClients().put(remoteCluster, client);
PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService);
doReturn(new CompletableFuture<Producer>()).when(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
replicator.startProducer();
verify(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
replicator.disconnect(false);
replicator.disconnect(false);
replicator.startProducer();
verify(clientImpl, Mockito.times(2)).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.
the class ReplicatorTest method testReplicatorClearBacklog.
@Test
public void testReplicatorClearBacklog() throws Exception {
// This test is to verify that reset cursor fails on global topic
SortedSet<String> testDests = new TreeSet<String>();
final DestinationName dest = DestinationName.get("persistent://pulsar/global/ns/clearBacklogTopic");
testDests.add(dest.toString());
MessageProducer producer1 = new MessageProducer(url1, dest);
MessageConsumer consumer1 = new MessageConsumer(url3, dest);
// Produce from cluster1 and consume from the rest
producer1.produce(2);
producer1.close();
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
PersistentReplicator replicator = spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
replicator.clearBacklog().get();
Thread.sleep(100);
// for code-coverage
replicator.updateRates();
// for code-coverage
replicator.expireMessages(1);
ReplicatorStats status = replicator.getStats();
assertTrue(status.replicationBacklog == 0);
consumer1.close();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.
the class ReplicatorTest method testCloseReplicatorStartProducer.
/**
* It verifies that PersistentReplicator considers CursorAlreadyClosedException as non-retriable-read exception and
* it should closed the producer as cursor is already closed because replicator is already deleted.
*
* @throws Exception
*/
@Test(timeOut = 5000)
public void testCloseReplicatorStartProducer() throws Exception {
DestinationName dest = DestinationName.get("persistent://pulsar/global/ns1/closeCursor");
// Producer on r1
MessageProducer producer1 = new MessageProducer(url1, dest);
// Consumer on r1
MessageConsumer consumer1 = new MessageConsumer(url1, dest);
// Consumer on r2
MessageConsumer consumer2 = new MessageConsumer(url2, dest);
// Replicator for r1 -> r2
PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
PersistentReplicator replicator = topic.getPersistentReplicator("r2");
// close the cursor
Field cursorField = PersistentReplicator.class.getDeclaredField("cursor");
cursorField.setAccessible(true);
ManagedCursor cursor = (ManagedCursor) cursorField.get(replicator);
cursor.close();
// try to read entries
CountDownLatch latch = new CountDownLatch(1);
producer1.produce(10);
cursor.asyncReadEntriesOrWait(10, new ReadEntriesCallback() {
@Override
public void readEntriesComplete(List<Entry> entries, Object ctx) {
latch.countDown();
fail("it should have been failed");
}
@Override
public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
latch.countDown();
assertTrue(exception instanceof CursorAlreadyClosedException);
}
}, null);
// replicator-readException: cursorAlreadyClosed
replicator.readEntriesFailed(new CursorAlreadyClosedException("Cursor already closed exception"), null);
// wait replicator producer to be closed
Thread.sleep(1000);
// Replicator producer must be closed
Field producerField = PersistentReplicator.class.getDeclaredField("producer");
producerField.setAccessible(true);
ProducerImpl replicatorProducer = (ProducerImpl) producerField.get(replicator);
assertEquals(replicatorProducer, null);
producer1.close();
consumer1.close();
consumer2.close();
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.
the class PersistentTopics method skipAllMessages.
@POST
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/skip_all")
@ApiOperation(value = "Skip all messages on a topic subscription.", notes = "Completely clears the backlog on the subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic or subscription does not exist") })
public void skipAllMessages(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
destination = decode(destination);
DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
if (partitionMetadata.partitions > 0) {
try {
for (int i = 0; i < partitionMetadata.partitions; i++) {
pulsar().getAdminClient().persistentTopics().skipAllMessages(dn.getPartition(i).toString(), subName);
}
} catch (Exception e) {
throw new RestException(e);
}
} else {
validateAdminOperationOnDestination(dn, authoritative);
PersistentTopic topic = getTopicReference(dn);
try {
if (subName.startsWith(topic.replicatorPrefix)) {
String remoteCluster = PersistentReplicator.getRemoteCluster(subName);
PersistentReplicator repl = topic.getPersistentReplicator(remoteCluster);
checkNotNull(repl);
repl.clearBacklog().get();
} else {
PersistentSubscription sub = topic.getPersistentSubscription(subName);
checkNotNull(sub);
sub.clearBacklog().get();
}
log.info("[{}] Cleared backlog on {} {}", clientAppId(), dn, subName);
} catch (NullPointerException npe) {
throw new RestException(Status.NOT_FOUND, "Subscription not found");
} catch (Exception exception) {
log.error("[{}] Failed to skip all messages {} {}", clientAppId(), dn, subName, exception);
throw new RestException(exception);
}
}
}
use of com.yahoo.pulsar.broker.service.persistent.PersistentReplicator in project pulsar by yahoo.
the class PersistentTopics method getReplicatorReference.
/**
* Get the Replicator object reference from the Topic reference
*/
private PersistentReplicator getReplicatorReference(String replName, PersistentTopic topic) {
try {
String remoteCluster = PersistentReplicator.getRemoteCluster(replName);
PersistentReplicator repl = topic.getPersistentReplicator(remoteCluster);
return checkNotNull(repl);
} catch (Exception e) {
throw new RestException(Status.NOT_FOUND, "Replicator not found");
}
}
Aggregations