Examples with RetriableException org.apache.kafka.common.errors.RetriableException used on opensource projects

Example 6 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project kafka by apache.

the class Fetcher method resetOffsetsAsync.

private void resetOffsetsAsync(Map<TopicPartition, Long> partitionResetTimestamps) {
    Map<Node, Map<TopicPartition, ListOffsetsPartition>> timestampsToSearchByNode = groupListOffsetRequests(partitionResetTimestamps, new HashSet<>());
    for (Map.Entry<Node, Map<TopicPartition, ListOffsetsPartition>> entry : timestampsToSearchByNode.entrySet()) {
        Node node = entry.getKey();
        final Map<TopicPartition, ListOffsetsPartition> resetTimestamps = entry.getValue();
        subscriptions.setNextAllowedRetry(resetTimestamps.keySet(), time.milliseconds() + requestTimeoutMs);
        RequestFuture<ListOffsetResult> future = sendListOffsetRequest(node, resetTimestamps, false);
        future.addListener(new RequestFutureListener<ListOffsetResult>() {

            @Override
            public void onSuccess(ListOffsetResult result) {
                if (!result.partitionsToRetry.isEmpty()) {
                    subscriptions.requestFailed(result.partitionsToRetry, time.milliseconds() + retryBackoffMs);
                    metadata.requestUpdate();
                }
                for (Map.Entry<TopicPartition, ListOffsetData> fetchedOffset : result.fetchedOffsets.entrySet()) {
                    TopicPartition partition = fetchedOffset.getKey();
                    ListOffsetData offsetData = fetchedOffset.getValue();
                    ListOffsetsPartition requestedReset = resetTimestamps.get(partition);
                    resetOffsetIfNeeded(partition, timestampToOffsetResetStrategy(requestedReset.timestamp()), offsetData);
                }
            }

            @Override
            public void onFailure(RuntimeException e) {
                subscriptions.requestFailed(resetTimestamps.keySet(), time.milliseconds() + retryBackoffMs);
                metadata.requestUpdate();
                if (!(e instanceof RetriableException) && !cachedListOffsetsException.compareAndSet(null, e))
                    log.error("Discarding error in ListOffsetResponse because another error is pending", e);
            }
        });
    }
}

Example 7 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project kafka by apache.

the class RecordCollectorImpl method recordSendError.

private void recordSendError(final String topic, final Exception exception, final ProducerRecord<byte[], byte[]> serializedRecord) {
    String errorMessage = String.format(SEND_EXCEPTION_MESSAGE, topic, taskId, exception.toString());
    if (isFatalException(exception)) {
        errorMessage += "\nWritten offsets would not be recorded and no more records would be sent since this is a fatal error.";
        sendException.set(new StreamsException(errorMessage, exception));
    } else if (exception instanceof ProducerFencedException || exception instanceof InvalidProducerEpochException || exception instanceof OutOfOrderSequenceException) {
        errorMessage += "\nWritten offsets would not be recorded and no more records would be sent since the producer is fenced, " + "indicating the task may be migrated out";
        sendException.set(new TaskMigratedException(errorMessage, exception));
    } else {
        if (exception instanceof RetriableException) {
            errorMessage += "\nThe broker is either slow or in bad state (like not having enough replicas) in responding the request, " + "or the connection to broker was interrupted sending the request or receiving the response. " + "\nConsider overwriting `max.block.ms` and /or " + "`delivery.timeout.ms` to a larger value to wait longer for such scenarios and avoid timeout errors";
            sendException.set(new TaskCorruptedException(Collections.singleton(taskId)));
        } else {
            if (productionExceptionHandler.handle(serializedRecord, exception) == ProductionExceptionHandlerResponse.FAIL) {
                errorMessage += "\nException handler choose to FAIL the processing, no more records would be sent.";
                sendException.set(new StreamsException(errorMessage, exception));
            } else {
                errorMessage += "\nException handler choose to CONTINUE processing in spite of this error but written offsets would not be recorded.";
                droppedRecordsSensor.record();
            }
        }
    }
    log.error(errorMessage, exception);
}

Example 8 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project kafka by apache.

the class Fetcher method getTopicMetadata.

/**
     * Get metadata for all topics present in Kafka cluster
     *
     * @param request The MetadataRequest to send
     * @param timeout time for which getting topic metadata is attempted
     * @return The map of topics with their partition information
     */
public Map<String, List<PartitionInfo>> getTopicMetadata(MetadataRequest.Builder request, long timeout) {
    // Save the round trip if no topics are requested.
    if (!request.isAllTopics() && request.topics().isEmpty())
        return Collections.emptyMap();
    long start = time.milliseconds();
    long remaining = timeout;
    do {
        RequestFuture<ClientResponse> future = sendMetadataRequest(request);
        client.poll(future, remaining);
        if (future.failed() && !future.isRetriable())
            throw future.exception();
        if (future.succeeded()) {
            MetadataResponse response = (MetadataResponse) future.value().responseBody();
            Cluster cluster = response.cluster();
            Set<String> unauthorizedTopics = cluster.unauthorizedTopics();
            if (!unauthorizedTopics.isEmpty())
                throw new TopicAuthorizationException(unauthorizedTopics);
            boolean shouldRetry = false;
            Map<String, Errors> errors = response.errors();
            if (!errors.isEmpty()) {
                // if there were errors, we need to check whether they were fatal or whether
                // we should just retry
                log.debug("Topic metadata fetch included errors: {}", errors);
                for (Map.Entry<String, Errors> errorEntry : errors.entrySet()) {
                    String topic = errorEntry.getKey();
                    Errors error = errorEntry.getValue();
                    if (error == Errors.INVALID_TOPIC_EXCEPTION)
                        throw new InvalidTopicException("Topic '" + topic + "' is invalid");
                    else if (error == Errors.UNKNOWN_TOPIC_OR_PARTITION)
                        // in the returned map
                        continue;
                    else if (error.exception() instanceof RetriableException)
                        shouldRetry = true;
                    else
                        throw new KafkaException("Unexpected error fetching metadata for topic " + topic, error.exception());
                }
            }
            if (!shouldRetry) {
                HashMap<String, List<PartitionInfo>> topicsPartitionInfos = new HashMap<>();
                for (String topic : cluster.topics()) topicsPartitionInfos.put(topic, cluster.availablePartitionsForTopic(topic));
                return topicsPartitionInfos;
            }
        }
        long elapsed = time.milliseconds() - start;
        remaining = timeout - elapsed;
        if (remaining > 0) {
            long backoff = Math.min(remaining, retryBackoffMs);
            time.sleep(backoff);
            remaining -= backoff;
        }
    } while (remaining > 0);
    throw new TimeoutException("Timeout expired while fetching topic metadata");
}

Example 9 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project divolte-collector by divolte.

the class KafkaFlusher method sendBatch.

@Override
protected ImmutableList<ProducerRecord<DivolteIdentifier, AvroRecordBuffer>> sendBatch(final List<ProducerRecord<DivolteIdentifier, AvroRecordBuffer>> batch) throws InterruptedException {
    // First start sending the messages.
    // (This will serialize them, determine the partition and then assign them to a per-partition buffer.)
    final int batchSize = batch.size();
    final List<Future<RecordMetadata>> sendResults = batch.stream().map(producer::send).collect(Collectors.toCollection(() -> new ArrayList<>(batchSize)));
    // Force a flush so we can check the results without blocking unnecessarily due to
    // a user-configured flushing policy.
    producer.flush();
    // When finished, each message can be in one of several states.
    // - Completed.
    // - An error occurred, but a retry may succeed.
    // - A fatal error occurred.
    // (In addition, we can be interrupted due to shutdown.)
    final ImmutableList.Builder<ProducerRecord<DivolteIdentifier, AvroRecordBuffer>> remaining = ImmutableList.builder();
    for (int i = 0; i < batchSize; ++i) {
        final Future<RecordMetadata> result = sendResults.get(i);
        try {
            final RecordMetadata metadata = result.get();
            if (logger.isDebugEnabled()) {
                final ProducerRecord<DivolteIdentifier, AvroRecordBuffer> record = batch.get(i);
                logger.debug("Finished sending event (partyId={}) to Kafka: topic/partition/offset = {}/{}/{}", record.key(), metadata.topic(), metadata.partition(), metadata.offset());
            }
        } catch (final ExecutionException e) {
            final Throwable cause = e.getCause();
            final ProducerRecord<DivolteIdentifier, AvroRecordBuffer> record = batch.get(i);
            if (cause instanceof RetriableException) {
                // A retry may succeed.
                if (logger.isDebugEnabled()) {
                    logger.debug("Transient error sending event (partyId=" + record.key() + ") to Kafka. Will retry.", cause);
                }
                remaining.add(record);
            } else {
                // Fatal error.
                logger.error("Error sending event (partyId=" + record.key() + ") to Kafka; abandoning.", cause);
            }
        }
    }
    return remaining.build();
}

Example 10 with RetriableException

use of org.apache.kafka.common.errors.RetriableException in project apache-kafka-on-k8s by banzaicloud.

the class RecordCollectorImpl method recordSendError.

private <K, V> void recordSendError(final K key, final V value, final Long timestamp, final String topic, final Exception exception) {
    String errorLogMessage = LOG_MESSAGE;
    String errorMessage = EXCEPTION_MESSAGE;
    if (exception instanceof RetriableException) {
        errorLogMessage += PARAMETER_HINT;
        errorMessage += PARAMETER_HINT;
    }
    log.error(errorLogMessage, key, value, timestamp, topic, exception.toString());
    sendException = new StreamsException(String.format(errorMessage, logPrefix, "an error caught", key, value, timestamp, topic, exception.getMessage()), exception);
}