Examples with PersistentReplicator org.apache.pulsar.broker.service.persistent.PersistentReplicator used on opensource projects

Example 6 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalSkipAllMessages.

protected void internalSkipAllMessages(String subName, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        try {
            for (int i = 0; i < partitionMetadata.partitions; i++) {
                pulsar().getAdminClient().persistentTopics().skipAllMessages(topicName.getPartition(i).toString(), subName);
            }
        } catch (Exception e) {
            throw new RestException(e);
        }
    } else {
        validateAdminOperationOnTopic(authoritative);
        PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
        try {
            if (subName.startsWith(topic.replicatorPrefix)) {
                String remoteCluster = PersistentReplicator.getRemoteCluster(subName);
                PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster);
                checkNotNull(repl);
                repl.clearBacklog().get();
            } else {
                PersistentSubscription sub = topic.getSubscription(subName);
                checkNotNull(sub);
                sub.clearBacklog().get();
            }
            log.info("[{}] Cleared backlog on {} {}", clientAppId(), topicName, subName);
        } catch (NullPointerException npe) {
            throw new RestException(Status.NOT_FOUND, "Subscription not found");
        } catch (Exception exception) {
            log.error("[{}] Failed to skip all messages {} {}", clientAppId(), topicName, subName, exception);
            throw new RestException(exception);
        }
    }
}

Example 7 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicsBase method getReplicatorReference.

/**
 * Get the Replicator object reference from the Topic reference
 */
private PersistentReplicator getReplicatorReference(String replName, PersistentTopic topic) {
    try {
        String remoteCluster = PersistentReplicator.getRemoteCluster(replName);
        PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster);
        return checkNotNull(repl);
    } catch (Exception e) {
        throw new RestException(Status.NOT_FOUND, "Replicator not found");
    }
}

Example 8 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatePeekAndSkip.

@Test(timeOut = 30000)
public void testReplicatePeekAndSkip() throws Exception {
    SortedSet<String> testDests = new TreeSet<String>();
    final TopicName dest = TopicName.get("persistent://pulsar/global/ns/peekAndSeekTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) topic.getReplicators().get(topic.getReplicators().keys().get(0));
    replicator.skipMessages(2);
    CompletableFuture<Entry> result = replicator.peekNthMessage(1);
    Entry entry = result.get(50, TimeUnit.MILLISECONDS);
    assertNull(entry);
    consumer1.close();
}

Example 9 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class ReplicatorTest method testReplicatorClearBacklog.

@Test(timeOut = 30000)
public void testReplicatorClearBacklog() throws Exception {
    // This test is to verify that reset cursor fails on global topic
    SortedSet<String> testDests = new TreeSet<String>();
    final TopicName dest = TopicName.get("persistent://pulsar/global/ns/clearBacklogTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = (PersistentReplicator) spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
    replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    replicator.clearBacklog().get();
    Thread.sleep(100);
    // for code-coverage
    replicator.updateRates();
    // for code-coverage
    replicator.expireMessages(1);
    ReplicatorStats status = replicator.getStats();
    assertTrue(status.replicationBacklog == 0);
    consumer1.close();
}

Example 10 with PersistentReplicator

use of org.apache.pulsar.broker.service.persistent.PersistentReplicator in project incubator-pulsar by apache.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
 * {@link NonPersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
 * 1. disconnect replicator producer
 * <p>
 * 2. close cursor
 * <p>
 * 3. remove from replicator-list.
 * <p>
 *
 * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
 *
 * @throws Exception
 */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, Replicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.builder().serviceUrl(brokerUrl.toString()).build();
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path(POLICIES, TopicName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}