Examples with PersistentSubscription org.apache.pulsar.broker.service.persistent.PersistentSubscription used on opensource projects

Example 16 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testMessageReplay.

/**
 * Verify: 1. Broker should not replay already acknowledged messages 2. Dispatcher should not stuck while
 * dispatching new messages due to previous-replay of invalid/already-acked messages
 *
 * @throws Exception
 */
@Test
public void testMessageReplay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message<byte[]> msg;
    int totalMessages = 10;
    int replayIndex = totalMessages / 2;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(1).subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subRef.getDispatcher();
    Field replayMap = PersistentDispatcherMultipleConsumers.class.getDeclaredField("messagesToReplay");
    replayMap.setAccessible(true);
    ConcurrentLongPairSet messagesToReplay = new ConcurrentLongPairSet(64, 1);
    assertNotNull(subRef);
    // (1) Produce messages
    for (int i = 0; i < totalMessages; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    MessageIdImpl firstAckedMsg = null;
    // (2) Consume and ack messages except first message
    for (int i = 0; i < totalMessages; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
        MessageIdImpl msgId = (MessageIdImpl) msg.getMessageId();
        if (i == 0) {
            firstAckedMsg = msgId;
        }
        if (i < replayIndex) {
            // (3) accumulate acked messages for replay
            messagesToReplay.add(msgId.getLedgerId(), msgId.getEntryId());
        }
    }
    // (4) redelivery : should redeliver only unacked messages
    Thread.sleep(1000);
    replayMap.set(dispatcher, messagesToReplay);
    // (a) redelivery with all acked-message should clear messageReply bucket
    dispatcher.redeliverUnacknowledgedMessages(dispatcher.getConsumers().get(0));
    assertEquals(messagesToReplay.size(), 0);
    // (b) fill messageReplyBucket with already acked entry again: and try to publish new msg and read it
    messagesToReplay.add(firstAckedMsg.getLedgerId(), firstAckedMsg.getEntryId());
    replayMap.set(dispatcher, messagesToReplay);
    // send new message
    final String testMsg = "testMsg";
    producer.send(testMsg.getBytes());
    // consumer should be able to receive only new message and not the
    dispatcher.consumerFlow(dispatcher.getConsumers().get(0), 1);
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertNotNull(msg);
    assertEquals(msg.getData(), testMsg.getBytes());
    consumer.close();
    producer.close();
}

Example 17 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testConcurrentConsumerThreads.

// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
    // test concurrent consumer threads on same consumerId
    final String topicName = "persistent://prop/use/ns-abc/topic3";
    final String subName = "sub3";
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(recvQueueSize).subscribe();
                for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
                    Message<byte[]> msg = consumer.receive();
                    consumer.acknowledge(msg);
                }
                return null;
            }
        });
    }
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    barrier.await();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    // 1. cumulatively all threads drain the backlog
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 2. flow control works the same as single consumer single thread
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
    executor.shutdown();
}

Example 18 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalResetCursor.

protected void internalResetCursor(String subName, long timestamp, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        int numParts = partitionMetadata.partitions;
        int numPartException = 0;
        Exception partitionException = null;
        try {
            for (int i = 0; i < numParts; i++) {
                pulsar().getAdminClient().persistentTopics().resetCursor(topicName.getPartition(i).toString(), subName, timestamp);
            }
        } catch (PreconditionFailedException pfe) {
            // throw the last exception if all partitions get this error
            // any other exception on partition is reported back to user
            ++numPartException;
            partitionException = pfe;
        } catch (Exception e) {
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, e);
            throw new RestException(e);
        }
        // report an error to user if unable to reset for all partitions
        if (numPartException == numParts) {
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, partitionException);
            throw new RestException(Status.PRECONDITION_FAILED, partitionException.getMessage());
        } else if (numPartException > 0) {
            log.warn("[{}][{}] partial errors for reset cursor on subscription {} to time {} - ", clientAppId(), topicName, subName, timestamp, partitionException);
        }
    } else {
        validateAdminOperationOnTopic(authoritative);
        log.info("[{}][{}] received reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp);
        PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
        if (topic == null) {
            throw new RestException(Status.NOT_FOUND, "Topic not found");
        }
        try {
            PersistentSubscription sub = topic.getSubscription(subName);
            checkNotNull(sub);
            sub.resetCursor(timestamp).get();
            log.info("[{}][{}] reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp);
        } catch (Exception e) {
            Throwable t = e.getCause();
            log.warn("[{}] [{}] Failed to reset cursor on subscription {} to time {}", clientAppId(), topicName, subName, timestamp, e);
            if (e instanceof NullPointerException) {
                throw new RestException(Status.NOT_FOUND, "Subscription not found");
            } else if (e instanceof NotAllowedException) {
                throw new RestException(Status.METHOD_NOT_ALLOWED, e.getMessage());
            } else if (t instanceof SubscriptionInvalidCursorPosition) {
                throw new RestException(Status.PRECONDITION_FAILED, "Unable to find position for timestamp specified -" + t.getMessage());
            } else {
                throw new RestException(e);
            }
        }
    }
}

Example 19 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalSkipAllMessages.

protected void internalSkipAllMessages(String subName, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        try {
            for (int i = 0; i < partitionMetadata.partitions; i++) {
                pulsar().getAdminClient().persistentTopics().skipAllMessages(topicName.getPartition(i).toString(), subName);
            }
        } catch (Exception e) {
            throw new RestException(e);
        }
    } else {
        validateAdminOperationOnTopic(authoritative);
        PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
        try {
            if (subName.startsWith(topic.replicatorPrefix)) {
                String remoteCluster = PersistentReplicator.getRemoteCluster(subName);
                PersistentReplicator repl = (PersistentReplicator) topic.getPersistentReplicator(remoteCluster);
                checkNotNull(repl);
                repl.clearBacklog().get();
            } else {
                PersistentSubscription sub = topic.getSubscription(subName);
                checkNotNull(sub);
                sub.clearBacklog().get();
            }
            log.info("[{}] Cleared backlog on {} {}", clientAppId(), topicName, subName);
        } catch (NullPointerException npe) {
            throw new RestException(Status.NOT_FOUND, "Subscription not found");
        } catch (Exception exception) {
            log.error("[{}] Failed to skip all messages {} {}", clientAppId(), topicName, subName, exception);
            throw new RestException(exception);
        }
    }
}

Example 20 with PersistentSubscription

use of org.apache.pulsar.broker.service.persistent.PersistentSubscription in project incubator-pulsar by apache.

the class PersistentTopicTest method testCompactorSubscription.

@Test
public void testCompactorSubscription() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    CompactedTopic compactedTopic = mock(CompactedTopic.class);
    PersistentSubscription sub = new CompactorSubscription(topic, compactedTopic, Compactor.COMPACTION_SUBSCRIPTION, cursorMock);
    PositionImpl position = new PositionImpl(1, 1);
    long ledgerId = 0xc0bfefeL;
    sub.acknowledgeMessage(position, AckType.Cumulative, ImmutableMap.of(Compactor.COMPACTED_TOPIC_LEDGER_PROPERTY, ledgerId));
    verify(compactedTopic, Mockito.times(1)).newCompactedLedger(position, ledgerId);
}