Examples with ManagedCursor org.apache.bookkeeper.mledger.ManagedCursor used on opensource projects

Example 96 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class PersistentTopicTest method testAtomicReplicationRemoval.

/**
     * {@link PersistentReplicator.removeReplicator} doesn't remove replicator in atomic way and does in multiple step:
     * 1. disconnect replicator producer
     * <p>
     * 2. close cursor
     * <p>
     * 3. remove from replicator-list.
     * <p>
     * 
     * If we try to startReplicationProducer before step-c finish then it should not avoid restarting repl-producer.
     * 
     * @throws Exception
     */
@Test
public void testAtomicReplicationRemoval() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns-abc/successTopic";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + remoteCluster;
    ConcurrentOpenHashMap<String, PersistentReplicator> replicatorMap = topic.getReplicators();
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = PulsarClient.create(brokerUrl.toString());
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = spy(new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService));
    replicatorMap.put(remoteReplicatorName, replicator);
    // step-1 remove replicator : it will disconnect the producer but it will wait for callback to be completed
    Method removeMethod = PersistentTopic.class.getDeclaredMethod("removeReplicator", String.class);
    removeMethod.setAccessible(true);
    removeMethod.invoke(topic, remoteReplicatorName);
    // step-2 now, policies doesn't have removed replication cluster so, it should not invoke "startProducer" of the
    // replicator
    when(pulsar.getConfigurationCache().policiesCache().get(AdminResource.path("policies", DestinationName.get(globalTopicName).getNamespace()))).thenReturn(Optional.of(new Policies()));
    // try to start replicator again
    topic.startReplProducers();
    // verify: replicator.startProducer is not invoked
    verify(replicator, Mockito.times(0)).startProducer();
    // step-3 : complete the callback to remove replicator from the list
    ArgumentCaptor<DeleteCursorCallback> captor = ArgumentCaptor.forClass(DeleteCursorCallback.class);
    Mockito.verify(ledgerMock).asyncDeleteCursor(anyObject(), captor.capture(), anyObject());
    DeleteCursorCallback callback = captor.getValue();
    callback.deleteCursorComplete(null);
}

Example 97 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class PersistentTopicTest method testClosingReplicationProducerTwice.

@Test
public void testClosingReplicationProducerTwice() throws Exception {
    final String globalTopicName = "persistent://prop/global/ns/testClosingReplicationProducerTwice";
    String localCluster = "local";
    String remoteCluster = "remote";
    final ManagedLedger ledgerMock = mock(ManagedLedger.class);
    doNothing().when(ledgerMock).asyncDeleteCursor(anyObject(), anyObject(), anyObject());
    doReturn(new ArrayList<Object>()).when(ledgerMock).getCursors();
    PersistentTopic topic = new PersistentTopic(globalTopicName, ledgerMock, brokerService);
    String remoteReplicatorName = topic.replicatorPrefix + "." + localCluster;
    final URL brokerUrl = new URL("http://" + pulsar.getAdvertisedAddress() + ":" + pulsar.getConfiguration().getBrokerServicePort());
    PulsarClient client = spy(PulsarClient.create(brokerUrl.toString()));
    PulsarClientImpl clientImpl = (PulsarClientImpl) client;
    Field conf = PersistentReplicator.class.getDeclaredField("producerConfiguration");
    conf.setAccessible(true);
    ManagedCursor cursor = mock(ManagedCursorImpl.class);
    doReturn(remoteCluster).when(cursor).getName();
    brokerService.getReplicationClients().put(remoteCluster, client);
    PersistentReplicator replicator = new PersistentReplicator(topic, cursor, localCluster, remoteCluster, brokerService);
    doReturn(new CompletableFuture<Producer>()).when(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.startProducer();
    verify(clientImpl).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
    replicator.disconnect(false);
    replicator.disconnect(false);
    replicator.startProducer();
    verify(clientImpl, Mockito.times(2)).createProducerAsync(globalTopicName, (ProducerConfiguration) conf.get(replicator), remoteReplicatorName);
}

Example 98 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class PersistentTopicTest method testDispatcherSingleConsumerReadFailed.

@Test
public void testDispatcherSingleConsumerReadFailed() throws Exception {
    PersistentTopic topic = spy(new PersistentTopic(successTopicName, ledgerMock, brokerService));
    ManagedCursor cursor = mock(ManagedCursor.class);
    when(cursor.getName()).thenReturn("cursor");
    PersistentDispatcherSingleActiveConsumer dispatcher = new PersistentDispatcherSingleActiveConsumer(cursor, SubType.Exclusive, 1, topic);
    Consumer consumer = mock(Consumer.class);
    dispatcher.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), consumer);
    verify(topic, atLeast(1)).getBrokerService();
}

Example 99 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class PersistentTopic method subscribe.

@Override
public CompletableFuture<Consumer> subscribe(final ServerCnx cnx, String subscriptionName, long consumerId, SubType subType, int priorityLevel, String consumerName) {
    final CompletableFuture<Consumer> future = new CompletableFuture<>();
    if (hasBatchMessagePublished && !cnx.isBatchMessageCompatibleVersion()) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Consumer doesn't support batch-message {}", topic, subscriptionName);
        }
        future.completeExceptionally(new UnsupportedVersionException("Consumer doesn't support batch-message"));
        return future;
    }
    if (subscriptionName.startsWith(replicatorPrefix)) {
        log.warn("[{}] Failed to create subscription for {}", topic, subscriptionName);
        future.completeExceptionally(new NamingException("Subscription with reserved subscription name attempted"));
        return future;
    }
    lock.readLock().lock();
    try {
        if (isFenced) {
            log.warn("[{}] Attempting to subscribe to a fenced topic", topic);
            future.completeExceptionally(new TopicFencedException("Topic is temporarily unavailable"));
            return future;
        }
        USAGE_COUNT_UPDATER.incrementAndGet(this);
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Added consumer -- count: {}", topic, subscriptionName, consumerName, USAGE_COUNT_UPDATER.get(this));
        }
    } finally {
        lock.readLock().unlock();
    }
    ledger.asyncOpenCursor(Codec.encode(subscriptionName), new OpenCursorCallback() {

        @Override
        public void openCursorComplete(ManagedCursor cursor, Object ctx) {
            if (log.isDebugEnabled()) {
                log.debug("[{}][{}] Opened cursor for {} {}", topic, subscriptionName, consumerId, consumerName);
            }
            try {
                PersistentSubscription subscription = subscriptions.computeIfAbsent(subscriptionName, name -> new PersistentSubscription(PersistentTopic.this, cursor));
                Consumer consumer = new Consumer(subscription, subType, consumerId, priorityLevel, consumerName, brokerService.pulsar().getConfiguration().getMaxUnackedMessagesPerConsumer(), cnx, cnx.getRole());
                subscription.addConsumer(consumer);
                if (!cnx.isActive()) {
                    consumer.close();
                    if (log.isDebugEnabled()) {
                        log.debug("[{}] [{}] [{}] Subscribe failed -- count: {}", topic, subscriptionName, consumer.consumerName(), USAGE_COUNT_UPDATER.get(PersistentTopic.this));
                    }
                    future.completeExceptionally(new BrokerServiceException("Connection was closed while the opening the cursor "));
                } else {
                    log.info("[{}][{}] Created new subscription for {}", topic, subscriptionName, consumerId);
                    future.complete(consumer);
                }
            } catch (BrokerServiceException e) {
                if (e instanceof ConsumerBusyException) {
                    log.warn("[{}][{}] Consumer {} {} already connected", topic, subscriptionName, consumerId, consumerName);
                } else if (e instanceof SubscriptionBusyException) {
                    log.warn("[{}][{}] {}", topic, subscriptionName, e.getMessage());
                }
                USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
                future.completeExceptionally(e);
            }
        }

        @Override
        public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
            log.warn("[{}] Failed to create subscription for {}", topic, subscriptionName);
            USAGE_COUNT_UPDATER.decrementAndGet(PersistentTopic.this);
            future.completeExceptionally(new PersistenceException(exception));
        }
    }, null);
    return future;
}

Example 100 with ManagedCursor

use of org.apache.bookkeeper.mledger.ManagedCursor in project pulsar by yahoo.

the class ReplicatorTest method testCloseReplicatorStartProducer.

/**
     * It verifies that PersistentReplicator considers CursorAlreadyClosedException as non-retriable-read exception and
     * it should closed the producer as cursor is already closed because replicator is already deleted.
     * 
     * @throws Exception
     */
@Test(timeOut = 5000)
public void testCloseReplicatorStartProducer() throws Exception {
    DestinationName dest = DestinationName.get("persistent://pulsar/global/ns1/closeCursor");
    // Producer on r1
    MessageProducer producer1 = new MessageProducer(url1, dest);
    // Consumer on r1
    MessageConsumer consumer1 = new MessageConsumer(url1, dest);
    // Consumer on r2
    MessageConsumer consumer2 = new MessageConsumer(url2, dest);
    // Replicator for r1 -> r2
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = topic.getPersistentReplicator("r2");
    // close the cursor
    Field cursorField = PersistentReplicator.class.getDeclaredField("cursor");
    cursorField.setAccessible(true);
    ManagedCursor cursor = (ManagedCursor) cursorField.get(replicator);
    cursor.close();
    // try to read entries
    CountDownLatch latch = new CountDownLatch(1);
    producer1.produce(10);
    cursor.asyncReadEntriesOrWait(10, new ReadEntriesCallback() {

        @Override
        public void readEntriesComplete(List<Entry> entries, Object ctx) {
            latch.countDown();
            fail("it should have been failed");
        }

        @Override
        public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
            latch.countDown();
            assertTrue(exception instanceof CursorAlreadyClosedException);
        }
    }, null);
    // replicator-readException: cursorAlreadyClosed
    replicator.readEntriesFailed(new CursorAlreadyClosedException("Cursor already closed exception"), null);
    // wait replicator producer to be closed
    Thread.sleep(1000);
    // Replicator producer must be closed
    Field producerField = PersistentReplicator.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    ProducerImpl replicatorProducer = (ProducerImpl) producerField.get(replicator);
    assertEquals(replicatorProducer, null);
    producer1.close();
    consumer1.close();
    consumer2.close();
}