Examples with LatestConsumer io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer used on opensource projects

Example 1 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistry method close.

/**
 * Called when the server is shutting down.
 */
public synchronized void close() {
    if (closed) {
        LOG.warn("Already closed registry");
        return;
    }
    LOG.info("Closing scalable push registry for topic " + ksqlTopic.getKafkaTopicName());
    final LatestConsumer latestConsumer = this.latestConsumer.get();
    if (latestConsumer != null) {
        latestConsumer.closeAsync();
    }
    for (CatchupConsumer catchupConsumer : catchupConsumers.values()) {
        catchupConsumer.closeAsync();
    }
    catchupConsumers.clear();
    MoreExecutors.shutdownAndAwaitTermination(executorService, 5000, TimeUnit.MILLISECONDS);
    MoreExecutors.shutdownAndAwaitTermination(executorServiceCatchup, 5000, TimeUnit.MILLISECONDS);
    closed = true;
}

Example 2 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistry method startLatestIfNotRunning.

/**
 * Starts the Latest consumer if it's not started already.
 * @param processingQueue The queue to register with the latest consumer
 */
private synchronized void startLatestIfNotRunning(final Optional<ProcessingQueue> processingQueue) {
    final LatestConsumer latestConsumer = this.latestConsumer.get();
    if (latestConsumer != null && !latestConsumer.isClosed()) {
        return;
    }
    final LatestConsumer newLatestConsumer = createLatestConsumer(processingQueue);
    processingQueue.ifPresent(newLatestConsumer::register);
    this.latestConsumer.set(newLatestConsumer);
    executorService.submit(() -> runLatest(newLatestConsumer));
}

Example 3 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistry method register.

/**
 * Registers a ProcessingQueue with this scalable push registry so that it starts receiving
 * data as it streams in for that consumer.
 * @param processingQueue The queue to register
 * @param catchupMetadata The catchup metadata including offsets to start at
 */
public synchronized void register(final ProcessingQueue processingQueue, final Optional<CatchupMetadata> catchupMetadata) {
    if (closed) {
        throw new IllegalStateException("Shouldn't register after closing");
    }
    try {
        if (catchupMetadata.isPresent()) {
            if (catchupConsumers.size() >= ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_CATCHUP_CONSUMERS)) {
                processingQueue.onError();
                throw new KsqlException("Too many catchups registered, " + KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_CATCHUP_CONSUMERS + ":" + ksqlConfig.getInt(KsqlConfig.KSQL_QUERY_PUSH_V2_MAX_CATCHUP_CONSUMERS));
            }
            // Latest must be running for a catchup consumer to exist.
            startLatestIfNotRunning(Optional.empty());
            startCatchup(processingQueue, catchupMetadata.get());
        } else {
            final LatestConsumer latestConsumer = this.latestConsumer.get();
            if (latestConsumer != null && !latestConsumer.isClosed()) {
                latestConsumer.register(processingQueue);
            } else {
                // If latestConsumer is null, that means it's the first time.  If latestConsumer != null
                // but it's already closed, that means that it's stopping async, so we just let it
                // finish while creating a new one.
                startLatestIfNotRunning(Optional.of(processingQueue));
            }
        }
    } finally {
        // Make sure that if anything in the creation process throws an exception, that we stop the
        // latest if there are no existing queries.
        stopLatestConsumerOnLastRequest();
    }
}

Example 4 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistry method createLatestConsumer.

/**
 * Creates the latest consumer and its underlying kafka consumer.
 * @param processingQueue The queue on which to send an error if anything goes wrong
 * @return The new LatestConsumer
 */
private LatestConsumer createLatestConsumer(final Optional<ProcessingQueue> processingQueue) {
    KafkaConsumer<Object, GenericRow> consumer = null;
    LatestConsumer latestConsumer = null;
    try {
        consumer = kafkaConsumerFactory.create(ksqlTopic, logicalSchema, serviceContext, consumerProperties, ksqlConfig, getLatestConsumerGroupId());
        latestConsumer = latestConsumerFactory.create(ksqlTopic.getKafkaTopicName(), isWindowed(), logicalSchema, consumer, catchupCoordinator, this::catchupAssignmentUpdater, ksqlConfig, Clock.systemUTC());
        return latestConsumer;
    } catch (Exception e) {
        LOG.error("Couldn't create latest consumer", e);
        processingQueue.ifPresent(ProcessingQueue::onError);
        // We're not supposed to block here, but if it fails here, hopefully it can immediately close.
        if (consumer != null) {
            consumer.close();
        }
        throw e;
    }
}

Example 5 with LatestConsumer

use of io.confluent.ksql.physical.scalablepush.consumer.LatestConsumer in project ksql by confluentinc.

the class ScalablePushRegistry method unregister.

/**
 * Unregisters the given ProcessingQueue when its request is complete
 * @param processingQueue The queue to deregister
 */
public synchronized void unregister(final ProcessingQueue processingQueue) {
    if (closed) {
        throw new IllegalStateException("Shouldn't unregister after closing");
    }
    try {
        final LatestConsumer latestConsumer = this.latestConsumer.get();
        if (latestConsumer != null && !latestConsumer.isClosed()) {
            latestConsumer.unregister(processingQueue);
        }
        unregisterCatchup(processingQueue);
    } finally {
        // Make sure we stop latest after any unregisters if necessary. We don't except the above to
        // throw any exceptions, but it's wrapped in a finally to be safe.
        stopLatestConsumerOnLastRequest();
    }
}