Examples with Consumer com.yahoo.pulsar.broker.service.Consumer used on opensource projects

Example 1 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentDispatcherMultipleConsumers method readEntriesComplete.

@Override
public synchronized void readEntriesComplete(List<Entry> entries, Object ctx) {
    ReadType readType = (ReadType) ctx;
    int start = 0;
    int entriesToDispatch = entries.size();
    if (readType == ReadType.Normal) {
        havePendingRead = false;
    } else {
        havePendingReplayRead = false;
    }
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Increasing read batch size from {} to {}", name, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    if (shouldRewindBeforeReadingOrReplaying && readType == ReadType.Normal) {
        // All consumers got disconnected before the completion of the read operation
        entries.forEach(Entry::release);
        cursor.rewind();
        shouldRewindBeforeReadingOrReplaying = false;
        readMoreEntries();
        return;
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Distributing {} messages to {} consumers", name, entries.size(), consumerList.size());
    }
    while (entriesToDispatch > 0 && totalAvailablePermits > 0 && isAtleastOneConsumerAvailable()) {
        Consumer c = getNextConsumer();
        if (c == null) {
            // Do nothing, cursor will be rewind at reconnection
            entries.subList(start, entries.size()).forEach(Entry::release);
            cursor.rewind();
            return;
        }
        // round-robin dispatch batch size for this consumer
        int messagesForC = Math.min(Math.min(entriesToDispatch, c.getAvailablePermits()), MaxRoundRobinBatchSize);
        if (messagesForC > 0) {
            int msgSent = c.sendMessages(entries.subList(start, start + messagesForC)).getRight();
            if (readType == ReadType.Replay) {
                entries.subList(start, start + messagesForC).forEach(entry -> {
                    messagesToReplay.remove((PositionImpl) entry.getPosition());
                });
            }
            start += messagesForC;
            entriesToDispatch -= messagesForC;
            totalAvailablePermits -= msgSent;
        }
    }
    if (entriesToDispatch > 0) {
        if (log.isDebugEnabled()) {
            log.debug("[{}] No consumers found with available permits, storing {} positions for later replay", name, entries.size() - start);
        }
        entries.subList(start, entries.size()).forEach(entry -> {
            messagesToReplay.add((PositionImpl) entry.getPosition());
            entry.release();
        });
    }
    readMoreEntries();
}

Example 2 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentDispatcherSingleActiveConsumer method pickAndScheduleActiveConsumer.

private void pickAndScheduleActiveConsumer() {
    checkArgument(!consumers.isEmpty());
    consumers.sort((c1, c2) -> c1.consumerName().compareTo(c2.consumerName()));
    int index = partitionIndex % consumers.size();
    Consumer prevConsumer = ACTIVE_CONSUMER_UPDATER.getAndSet(this, consumers.get(index));
    if (prevConsumer == ACTIVE_CONSUMER_UPDATER.get(this)) {
        // Active consumer did not change. Do nothing at this point
        return;
    }
    if (havePendingRead && cursor.cancelPendingReadRequest()) {
        havePendingRead = false;
    }
    // let it finish and then rewind
    if (!havePendingRead) {
        cursor.rewind();
        readMoreEntries(ACTIVE_CONSUMER_UPDATER.get(this));
    }
}

Example 3 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentDispatcherSingleActiveConsumer method readEntriesComplete.

@Override
public synchronized void readEntriesComplete(final List<Entry> entries, Object obj) {
    Consumer readConsumer = (Consumer) obj;
    if (log.isDebugEnabled()) {
        log.debug("[{}] Got messages: {}", readConsumer, entries.size());
    }
    havePendingRead = false;
    if (readBatchSize < MaxReadBatchSize) {
        int newReadBatchSize = Math.min(readBatchSize * 2, MaxReadBatchSize);
        if (log.isDebugEnabled()) {
            log.debug("[{}] Increasing read batch size from {} to {}", readConsumer, readBatchSize, newReadBatchSize);
        }
        readBatchSize = newReadBatchSize;
    }
    readFailureBackoff.reduceToHalf();
    Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
    if (currentConsumer == null || readConsumer != currentConsumer) {
        // Active consumer has changed since the read request has been issued. We need to rewind the cursor and
        // re-issue the read request for the new consumer
        entries.forEach(Entry::release);
        cursor.rewind();
        if (currentConsumer != null) {
            readMoreEntries(currentConsumer);
        }
    } else {
        currentConsumer.sendMessages(entries).getLeft().addListener(future -> {
            if (future.isSuccess()) {
                synchronized (PersistentDispatcherSingleActiveConsumer.this) {
                    Consumer newConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
                    if (newConsumer != null && !havePendingRead) {
                        readMoreEntries(newConsumer);
                    } else {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] Ignoring write future complete. consumerAvailable={} havePendingRead={}", newConsumer, newConsumer != null, havePendingRead);
                        }
                    }
                }
            }
        });
    }
}

Example 4 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentDispatcherSingleActiveConsumer method readEntriesFailed.

@Override
public synchronized void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
    havePendingRead = false;
    Consumer c = (Consumer) ctx;
    long waitTimeMillis = readFailureBackoff.next();
    if (!(exception instanceof TooManyRequestsException)) {
        log.error("[{}] Error reading entries at {} : {} - Retrying to read in {} seconds", c, cursor.getReadPosition(), exception.getMessage(), waitTimeMillis / 1000.0);
    } else {
        if (log.isDebugEnabled()) {
            log.debug("[{}] Got throttled by bookies while reading at {} : {} - Retrying to read in {} seconds", c, cursor.getReadPosition(), exception.getMessage(), waitTimeMillis / 1000.0);
        }
    }
    checkNotNull(c);
    // Reduce read batch size to avoid flooding bookies with retries
    readBatchSize = 1;
    topic.getBrokerService().executor().schedule(() -> {
        synchronized (PersistentDispatcherSingleActiveConsumer.this) {
            Consumer currentConsumer = ACTIVE_CONSUMER_UPDATER.get(this);
            // we should retry the read if we have an active consumer and there is no pending read
            if (currentConsumer != null && !havePendingRead) {
                if (log.isDebugEnabled()) {
                    log.debug("[{}] Retrying read operation", c);
                }
                readMoreEntries(currentConsumer);
            } else {
                log.info("[{}] Skipping read retry: Current Consumer {}, havePendingRead {}", c, currentConsumer, havePendingRead);
            }
        }
    }, waitTimeMillis, TimeUnit.MILLISECONDS);
}

Example 5 with Consumer

use of com.yahoo.pulsar.broker.service.Consumer in project pulsar by yahoo.

the class PersistentTopicConcurrentTest method testConcurrentTopicDeleteAndUnsubscribe.

// @Test
public void testConcurrentTopicDeleteAndUnsubscribe() throws Exception {
    // create topic
    final PersistentTopic topic = (PersistentTopic) brokerService.getTopic(successTopicName).get();
    PulsarApi.CommandSubscribe cmd = PulsarApi.CommandSubscribe.newBuilder().setConsumerId(1).setTopic(successTopicName).setSubscription(successSubName).setRequestId(1).setSubType(PulsarApi.CommandSubscribe.SubType.Exclusive).build();
    Future<Consumer> f1 = topic.subscribe(serverCnx, cmd.getSubscription(), cmd.getConsumerId(), cmd.getSubType(), 0, cmd.getConsumerName());
    f1.get();
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    Thread deleter = new Thread() {

        public void run() {
            try {
                barrier.await();
                Thread.sleep(4, 700);
                log.info("deleter outcome is {}", topic.delete().get());
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread unsubscriber = new Thread() {

        public void run() {
            try {
                barrier.await();
                // Thread.sleep(2,0);
                // assertTrue(topic.unsubscribe(successSubName).isDone());
                ConcurrentOpenHashMap<String, PersistentSubscription> subscriptions = topic.getSubscriptions();
                PersistentSubscription ps = subscriptions.get(successSubName);
                log.info("unsubscribe result : {}", topic.unsubscribe(successSubName).get());
                log.info("closing consumer..");
                ps.getConsumers().get(0).close();
            } catch (Exception e) {
                e.printStackTrace();
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    deleter.start();
    unsubscriber.start();
    counter.await();
    assertEquals(gotException.get(), false);
}