Examples with RetriableException org.apache.kafka.common.errors.RetriableException used on opensource projects

Example 11 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method doAutoCommitOffsetsAsync.

private void doAutoCommitOffsetsAsync() {
    Map<TopicPartition, OffsetAndMetadata> allConsumedOffsets = subscriptions.allConsumed();
    log.debug("Sending asynchronous auto-commit of offsets {}", allConsumedOffsets);
    commitOffsetsAsync(allConsumedOffsets, new OffsetCommitCallback() {

        @Override
        public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
            if (exception != null) {
                if (exception instanceof RetriableException) {
                    log.debug("Asynchronous auto-commit of offsets {} failed due to retriable error: {}", offsets, exception);
                    nextAutoCommitDeadline = Math.min(time.milliseconds() + retryBackoffMs, nextAutoCommitDeadline);
                } else {
                    log.warn("Asynchronous auto-commit of offsets {} failed: {}", offsets, exception.getMessage());
                }
            } else {
                log.debug("Completed asynchronous auto-commit of offsets {}", offsets);
            }
        }
    });
}

Example 12 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinator method doCommitOffsetsAsync.

private void doCommitOffsetsAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, final OffsetCommitCallback callback) {
    RequestFuture<Void> future = sendOffsetCommitRequest(offsets);
    final OffsetCommitCallback cb = callback == null ? defaultOffsetCommitCallback : callback;
    future.addListener(new RequestFutureListener<Void>() {

        @Override
        public void onSuccess(Void value) {
            if (interceptors != null)
                interceptors.onCommit(offsets);
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, null));
        }

        @Override
        public void onFailure(RuntimeException e) {
            Exception commitException = e;
            if (e instanceof RetriableException)
                commitException = new RetriableCommitFailedException(e);
            completedOffsetCommits.add(new OffsetCommitCompletion(cb, offsets, commitException));
        }
    });
}

Example 13 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project apache-kafka-on-k8s by banzaicloud.

the class Fetcher method getTopicMetadata.

/**
 * Get metadata for all topics present in Kafka cluster
 *
 * @param request The MetadataRequest to send
 * @param timeout time for which getting topic metadata is attempted
 * @return The map of topics with their partition information
 */
public Map<String, List<PartitionInfo>> getTopicMetadata(MetadataRequest.Builder request, long timeout) {
    // Save the round trip if no topics are requested.
    if (!request.isAllTopics() && request.topics().isEmpty())
        return Collections.emptyMap();
    long start = time.milliseconds();
    long remaining = timeout;
    do {
        RequestFuture<ClientResponse> future = sendMetadataRequest(request);
        client.poll(future, remaining);
        if (future.failed() && !future.isRetriable())
            throw future.exception();
        if (future.succeeded()) {
            MetadataResponse response = (MetadataResponse) future.value().responseBody();
            Cluster cluster = response.cluster();
            Set<String> unauthorizedTopics = cluster.unauthorizedTopics();
            if (!unauthorizedTopics.isEmpty())
                throw new TopicAuthorizationException(unauthorizedTopics);
            boolean shouldRetry = false;
            Map<String, Errors> errors = response.errors();
            if (!errors.isEmpty()) {
                // if there were errors, we need to check whether they were fatal or whether
                // we should just retry
                log.debug("Topic metadata fetch included errors: {}", errors);
                for (Map.Entry<String, Errors> errorEntry : errors.entrySet()) {
                    String topic = errorEntry.getKey();
                    Errors error = errorEntry.getValue();
                    if (error == Errors.INVALID_TOPIC_EXCEPTION)
                        throw new InvalidTopicException("Topic '" + topic + "' is invalid");
                    else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION)
                        // in the returned map
                        continue;
                    else if (error.exception() instanceof RetriableException)
                        shouldRetry = true;
                    else
                        throw new KafkaException("Unexpected error fetching metadata for topic " + topic, error.exception());
                }
            }
            if (!shouldRetry) {
                HashMap<String, List<PartitionInfo>> topicsPartitionInfos = new HashMap<>();
                for (String topic : cluster.topics()) topicsPartitionInfos.put(topic, cluster.availablePartitionsForTopic(topic));
                return topicsPartitionInfos;
            }
        }
        long elapsed = time.milliseconds() - start;
        remaining = timeout - elapsed;
        if (remaining > 0) {
            long backoff = Math.min(remaining, retryBackoffMs);
            time.sleep(backoff);
            remaining -= backoff;
        }
    } while (remaining > 0);
    throw new TimeoutException("Timeout expired while fetching topic metadata");
}

Example 14 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project apache-kafka-on-k8s by banzaicloud.

the class Sender method maybeWaitForProducerId.

private void maybeWaitForProducerId() {
    while (!transactionManager.hasProducerId() && !transactionManager.hasError()) {
        try {
            Node node = awaitLeastLoadedNodeReady(requestTimeout);
            if (node != null) {
                ClientResponse response = sendAndAwaitInitProducerIdRequest(node);
                InitProducerIdResponse initProducerIdResponse = (InitProducerIdResponse) response.responseBody();
                Errors error = initProducerIdResponse.error();
                if (error == Errors.NONE) {
                    ProducerIdAndEpoch producerIdAndEpoch = new ProducerIdAndEpoch(initProducerIdResponse.producerId(), initProducerIdResponse.epoch());
                    transactionManager.setProducerIdAndEpoch(producerIdAndEpoch);
                    return;
                } else if (error.exception() instanceof RetriableException) {
                    log.debug("Retriable error from InitProducerId response", error.message());
                } else {
                    transactionManager.transitionToFatalError(error.exception());
                    break;
                }
            } else {
                log.debug("Could not find an available broker to send InitProducerIdRequest to. " + "We will back off and try again.");
            }
        } catch (UnsupportedVersionException e) {
            transactionManager.transitionToFatalError(e);
            break;
        } catch (IOException e) {
            log.debug("Broker {} disconnected while awaiting InitProducerId response", e);
        }
        log.trace("Retry InitProducerIdRequest in {}ms.", retryBackoffMs);
        time.sleep(retryBackoffMs);
        metadata.requestUpdate();
    }
}

Example 15 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project kafka by apache.

the class Fetcher method getTopicMetadata.

/**
 * Get metadata for all topics present in Kafka cluster
 *
 * @param request The MetadataRequest to send
 * @param timer Timer bounding how long this method can block
 * @return The map of topics with their partition information
 */
public Map<String, List<PartitionInfo>> getTopicMetadata(MetadataRequest.Builder request, Timer timer) {
    // Save the round trip if no topics are requested.
    if (!request.isAllTopics() && request.emptyTopicList())
        return Collections.emptyMap();
    do {
        RequestFuture<ClientResponse> future = sendMetadataRequest(request);
        client.poll(future, timer);
        if (future.failed() && !future.isRetriable())
            throw future.exception();
        if (future.succeeded()) {
            MetadataResponse response = (MetadataResponse) future.value().responseBody();
            Cluster cluster = response.buildCluster();
            Set<String> unauthorizedTopics = cluster.unauthorizedTopics();
            if (!unauthorizedTopics.isEmpty())
                throw new TopicAuthorizationException(unauthorizedTopics);
            boolean shouldRetry = false;
            Map<String, Errors> errors = response.errors();
            if (!errors.isEmpty()) {
                // if there were errors, we need to check whether they were fatal or whether
                // we should just retry
                log.debug("Topic metadata fetch included errors: {}", errors);
                for (Map.Entry<String, Errors> errorEntry : errors.entrySet()) {
                    String topic = errorEntry.getKey();
                    Errors error = errorEntry.getValue();
                    if (error == Errors.INVALID_TOPIC_EXCEPTION)
                        throw new InvalidTopicException("Topic '" + topic + "' is invalid");
                    else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION)
                        // in the returned map
                        continue;
                    else if (error.exception() instanceof RetriableException)
                        shouldRetry = true;
                    else
                        throw new KafkaException("Unexpected error fetching metadata for topic " + topic, error.exception());
                }
            }
            if (!shouldRetry) {
                HashMap<String, List<PartitionInfo>> topicsPartitionInfos = new HashMap<>();
                for (String topic : cluster.topics()) topicsPartitionInfos.put(topic, cluster.partitionsForTopic(topic));
                return topicsPartitionInfos;
            }
        }
        timer.sleep(retryBackoffMs);
    } while (timer.notExpired());
    throw new TimeoutException("Timeout expired while fetching topic metadata");
}