Examples with PulsarClientException org.apache.pulsar.client.api.PulsarClientException used on opensource projects

Example 46 with PulsarClientException

use of org.apache.pulsar.client.api.PulsarClientException in project incubator-pulsar by apache.

the class PartitionedConsumerImpl method start.

private void start() {
    AtomicReference<Throwable> subscribeFail = new AtomicReference<Throwable>();
    AtomicInteger completed = new AtomicInteger();
    ConsumerConfigurationData<T> internalConfig = getInternalConsumerConfig();
    for (int partitionIndex = 0; partitionIndex < numPartitions; partitionIndex++) {
        String partitionName = TopicName.get(topic).getPartition(partitionIndex).toString();
        ConsumerImpl<T> consumer = new ConsumerImpl<>(client, partitionName, internalConfig, client.externalExecutorProvider().getExecutor(), partitionIndex, new CompletableFuture<>(), schema);
        consumers.add(consumer);
        consumer.subscribeFuture().handle((cons, subscribeException) -> {
            if (subscribeException != null) {
                setState(State.Failed);
                subscribeFail.compareAndSet(null, subscribeException);
                client.cleanupConsumer(this);
            }
            if (completed.incrementAndGet() == numPartitions) {
                if (subscribeFail.get() == null) {
                    try {
                        // We have successfully created N consumers, so we can start receiving messages now
                        starReceivingMessages();
                        setState(State.Ready);
                        subscribeFuture().complete(PartitionedConsumerImpl.this);
                        log.info("[{}] [{}] Created partitioned consumer", topic, subscription);
                        return null;
                    } catch (PulsarClientException e) {
                        subscribeFail.set(e);
                    }
                }
                closeAsync().handle((ok, closeException) -> {
                    subscribeFuture().completeExceptionally(subscribeFail.get());
                    client.cleanupConsumer(this);
                    return null;
                });
                log.error("[{}] [{}] Could not create partitioned consumer.", topic, subscription, subscribeFail.get().getCause());
            }
            return null;
        });
    }
}

Example 47 with PulsarClientException

use of org.apache.pulsar.client.api.PulsarClientException in project incubator-pulsar by apache.

the class ProducerImpl method run.

/**
 * Process sendTimeout events
 */
@Override
public void run(Timeout timeout) throws Exception {
    if (timeout.isCancelled()) {
        return;
    }
    long timeToWaitMs;
    synchronized (this) {
        // If it's closing/closed we need to ignore this timeout and not schedule next timeout.
        if (getState() == State.Closing || getState() == State.Closed) {
            return;
        }
        OpSendMsg firstMsg = pendingMessages.peek();
        if (firstMsg == null) {
            // If there are no pending messages, reset the timeout to the configured value.
            timeToWaitMs = conf.getSendTimeoutMs();
        } else {
            // If there is at least one message, calculate the diff between the message timeout and the current
            // time.
            long diff = (firstMsg.createdAt + conf.getSendTimeoutMs()) - System.currentTimeMillis();
            if (diff <= 0) {
                // The diff is less than or equal to zero, meaning that the message has been timed out.
                // Set the callback to timeout on every message, then clear the pending queue.
                log.info("[{}] [{}] Message send timed out. Failing {} messages", topic, producerName, pendingMessages.size());
                PulsarClientException te = new PulsarClientException.TimeoutException("Could not send message to broker within given timeout");
                failPendingMessages(cnx(), te);
                stats.incrementSendFailed(pendingMessages.size());
                // Since the pending queue is cleared now, set timer to expire after configured value.
                timeToWaitMs = conf.getSendTimeoutMs();
            } else {
                // The diff is greater than zero, set the timeout to the diff value
                timeToWaitMs = diff;
            }
        }
        sendTimeout = client.timer().newTimeout(this, timeToWaitMs, TimeUnit.MILLISECONDS);
    }
}

Example 48 with PulsarClientException

use of org.apache.pulsar.client.api.PulsarClientException in project incubator-pulsar by apache.

the class TopicsConsumerImpl method subscribeAsync.

// subscribe one more given topic
public CompletableFuture<Void> subscribeAsync(String topicName) {
    if (!topicNameValid(topicName)) {
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topic name not valid"));
    }
    if (getState() == State.Closing || getState() == State.Closed) {
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Topics Consumer was already closed"));
    }
    CompletableFuture<Void> subscribeResult = new CompletableFuture<>();
    final AtomicInteger partitionNumber = new AtomicInteger(0);
    client.getPartitionedTopicMetadata(topicName).thenAccept(metadata -> {
        if (log.isDebugEnabled()) {
            log.debug("Received topic {} metadata.partitions: {}", topicName, metadata.partitions);
        }
        List<CompletableFuture<Consumer<T>>> futureList;
        if (metadata.partitions > 1) {
            this.topics.putIfAbsent(topicName, metadata.partitions);
            numberTopicPartitions.addAndGet(metadata.partitions);
            partitionNumber.addAndGet(metadata.partitions);
            futureList = IntStream.range(0, partitionNumber.get()).mapToObj(partitionIndex -> {
                String partitionName = TopicName.get(topicName).getPartition(partitionIndex).toString();
                CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
                ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, partitionName, internalConfig, client.externalExecutorProvider().getExecutor(), partitionIndex, subFuture, schema);
                consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
                return subFuture;
            }).collect(Collectors.toList());
        } else {
            this.topics.putIfAbsent(topicName, 1);
            numberTopicPartitions.incrementAndGet();
            partitionNumber.incrementAndGet();
            CompletableFuture<Consumer<T>> subFuture = new CompletableFuture<>();
            ConsumerImpl<T> newConsumer = new ConsumerImpl<>(client, topicName, internalConfig, client.externalExecutorProvider().getExecutor(), 0, subFuture, schema);
            consumers.putIfAbsent(newConsumer.getTopic(), newConsumer);
            futureList = Collections.singletonList(subFuture);
        }
        FutureUtil.waitForAll(futureList).thenAccept(finalFuture -> {
            try {
                if (numberTopicPartitions.get() > maxReceiverQueueSize) {
                    setMaxReceiverQueueSize(numberTopicPartitions.get());
                }
                int numTopics = this.topics.values().stream().mapToInt(Integer::intValue).sum();
                checkState(numberTopicPartitions.get() == numTopics, "numberTopicPartitions " + numberTopicPartitions.get() + " not equals expected: " + numTopics);
                // We have successfully created new consumers, so we can start receiving messages for them
                startReceivingMessages(consumers.values().stream().filter(consumer1 -> {
                    String consumerTopicName = consumer1.getTopic();
                    if (TopicName.get(consumerTopicName).getPartitionedTopicName().equals(TopicName.get(topicName).getPartitionedTopicName().toString())) {
                        return true;
                    } else {
                        return false;
                    }
                }).collect(Collectors.toList()));
                subscribeResult.complete(null);
                log.info("[{}] [{}] Success subscribe new topic {} in topics consumer, numberTopicPartitions {}", topic, subscription, topicName, numberTopicPartitions.get());
                if (this.namespaceName == null) {
                    this.namespaceName = TopicName.get(topicName).getNamespaceObject();
                }
                return;
            } catch (PulsarClientException e) {
                handleSubscribeOneTopicError(topicName, e);
                subscribeResult.completeExceptionally(e);
            }
        }).exceptionally(ex -> {
            handleSubscribeOneTopicError(topicName, ex);
            subscribeResult.completeExceptionally(ex);
            return null;
        });
    }).exceptionally(ex1 -> {
        log.warn("[{}] Failed to get partitioned topic metadata: {}", topicName, ex1.getMessage());
        subscribeResult.completeExceptionally(ex1);
        return null;
    });
    return subscribeResult;
}

Example 49 with PulsarClientException

use of org.apache.pulsar.client.api.PulsarClientException in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testSimpleConsumerEventsWithoutPartition.

@Test
public void testSimpleConsumerEventsWithoutPartition() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic1";
    final String subName = "sub1";
    final int numMsgs = 100;
    TestConsumerStateEventListener listener1 = new TestConsumerStateEventListener();
    TestConsumerStateEventListener listener2 = new TestConsumerStateEventListener();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover);
    // 1. two consumers on the same subscription
    ConsumerBuilder<byte[]> consumerBulder1 = consumerBuilder.clone().consumerName("1").consumerEventListener(listener1);
    Consumer<byte[]> consumer1 = consumerBulder1.subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().consumerName("2").consumerEventListener(listener2).subscribe();
    verifyConsumerActive(listener1, -1);
    verifyConsumerInactive(listener2, -1);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    assertNotNull(topicRef);
    assertNotNull(subRef);
    // 2. validate basic dispatcher state
    assertTrue(subRef.getDispatcher().isConsumerConnected());
    assertEquals(subRef.getDispatcher().getType(), SubType.Failover);
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    rolloverPerIntervalStats();
    assertEquals(subRef.getNumberOfEntriesInBacklog(), numMsgs);
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    // 3. consumer1 should have all the messages while consumer2 should have no messages
    Message<byte[]> msg = null;
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    for (int i = 0; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    // 4. messages deleted on individual acks
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 5. master consumer failure should resend unacked messages and new messages to another consumer
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    for (int i = 5; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
    // do not ack
    }
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    verifyConsumerActive(listener2, -1);
    verifyConsumerNotReceiveAnyStateChanges(listener1);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    Assert.assertNull(consumer2.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 6. consumer subscription should send messages to the new consumer if its name is highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer2.acknowledge(msg);
    }
    consumer1 = consumerBulder1.subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    Assert.assertNull(consumer1.receive(1, TimeUnit.SECONDS));
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    // 7. consumer subscription should not send messages to the new consumer if its name is not highest in the list
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    TestConsumerStateEventListener listener3 = new TestConsumerStateEventListener();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().consumerName("3").consumerEventListener(listener3).subscribe();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    verifyConsumerInactive(listener3, -1);
    Assert.assertNull(consumer3.receive(1, TimeUnit.SECONDS));
    for (int i = 5; i < numMsgs; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        Assert.assertNotNull(msg);
        Assert.assertEquals(new String(msg.getData()), "my-message-" + i);
        consumer1.acknowledge(msg);
    }
    rolloverPerIntervalStats();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 8. unsubscribe not allowed if multiple consumers connected
    try {
        consumer1.unsubscribe();
        fail("should fail");
    } catch (PulsarClientException e) {
    // ok
    }
    // 9. unsubscribe allowed if there is a lone consumer
    consumer1.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    consumer2.close();
    Thread.sleep(CONSUMER_ADD_OR_REMOVE_WAIT_TIME);
    try {
        consumer3.unsubscribe();
    } catch (PulsarClientException e) {
        fail("Should not fail", e);
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    subRef = topicRef.getSubscription(subName);
    assertNull(subRef);
    producer.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}

Example 50 with PulsarClientException

use of org.apache.pulsar.client.api.PulsarClientException in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testConsumersWithDifferentPermits.

@Test
public void testConsumersWithDifferentPermits() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic4";
    final String subName = "sub4";
    final int numMsgs = 10000;
    final AtomicInteger msgCountConsumer1 = new AtomicInteger(0);
    final AtomicInteger msgCountConsumer2 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs);
    int recvQ1 = 10;
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1).messageListener((consumer, msg) -> {
        msgCountConsumer1.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    int recvQ2 = 1;
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2).messageListener((consumer, msg) -> {
        msgCountConsumer2.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(5, TimeUnit.SECONDS);
    assertEquals(msgCountConsumer1.get(), numMsgs - numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    assertEquals(msgCountConsumer2.get(), numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}