Examples with InvalidMetadataException org.apache.kafka.common.errors.InvalidMetadataException used on opensource projects

Example 1 with InvalidMetadataException

use of org.apache.kafka.common.errors.InvalidMetadataException in project kafka by apache.

the class Metadata method handleMetadataResponse.

/**
 * Transform a MetadataResponse into a new MetadataCache instance.
 */
private MetadataCache handleMetadataResponse(MetadataResponse metadataResponse, boolean isPartialUpdate, long nowMs) {
    // All encountered topics.
    Set<String> topics = new HashSet<>();
    // Retained topics to be passed to the metadata cache.
    Set<String> internalTopics = new HashSet<>();
    Set<String> unauthorizedTopics = new HashSet<>();
    Set<String> invalidTopics = new HashSet<>();
    List<MetadataResponse.PartitionMetadata> partitions = new ArrayList<>();
    Map<String, Uuid> topicIds = new HashMap<>();
    Map<String, Uuid> oldTopicIds = cache.topicIds();
    for (MetadataResponse.TopicMetadata metadata : metadataResponse.topicMetadata()) {
        String topicName = metadata.topic();
        Uuid topicId = metadata.topicId();
        topics.add(topicName);
        // We can only reason about topic ID changes when both IDs are valid, so keep oldId null unless the new metadata contains a topic ID
        Uuid oldTopicId = null;
        if (!Uuid.ZERO_UUID.equals(topicId)) {
            topicIds.put(topicName, topicId);
            oldTopicId = oldTopicIds.get(topicName);
        } else {
            topicId = null;
        }
        if (!retainTopic(topicName, metadata.isInternal(), nowMs))
            continue;
        if (metadata.isInternal())
            internalTopics.add(topicName);
        if (metadata.error() == Errors.NONE) {
            for (MetadataResponse.PartitionMetadata partitionMetadata : metadata.partitionMetadata()) {
                // Even if the partition's metadata includes an error, we need to handle
                // the update to catch new epochs
                updateLatestMetadata(partitionMetadata, metadataResponse.hasReliableLeaderEpochs(), topicId, oldTopicId).ifPresent(partitions::add);
                if (partitionMetadata.error.exception() instanceof InvalidMetadataException) {
                    log.debug("Requesting metadata update for partition {} due to error {}", partitionMetadata.topicPartition, partitionMetadata.error);
                    requestUpdate();
                }
            }
        } else {
            if (metadata.error().exception() instanceof InvalidMetadataException) {
                log.debug("Requesting metadata update for topic {} due to error {}", topicName, metadata.error());
                requestUpdate();
            }
            if (metadata.error() == Errors.INVALID_TOPIC_EXCEPTION)
                invalidTopics.add(topicName);
            else if (metadata.error() == Errors.TOPIC_AUTHORIZATION_FAILED)
                unauthorizedTopics.add(topicName);
        }
    }
    Map<Integer, Node> nodes = metadataResponse.brokersById();
    if (isPartialUpdate)
        return this.cache.mergeWith(metadataResponse.clusterId(), nodes, partitions, unauthorizedTopics, invalidTopics, internalTopics, metadataResponse.controller(), topicIds, (topic, isInternal) -> !topics.contains(topic) && retainTopic(topic, isInternal, nowMs));
    else
        return new MetadataCache(metadataResponse.clusterId(), nodes, partitions, unauthorizedTopics, invalidTopics, internalTopics, metadataResponse.controller(), topicIds);
}

Example 2 with InvalidMetadataException

use of org.apache.kafka.common.errors.InvalidMetadataException in project kafka by apache.

the class Sender method completeBatch.

/**
 * Complete or retry the given batch of records.
 *
 * @param batch The record batch
 * @param response The produce response
 * @param correlationId The correlation id for the request
 * @param now The current POSIX timestamp in milliseconds
 */
private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, long correlationId, long now) {
    Errors error = response.error;
    if (error == Errors.MESSAGE_TOO_LARGE && batch.recordCount > 1 && !batch.isDone() && (batch.magic() >= RecordBatch.MAGIC_VALUE_V2 || batch.isCompressed())) {
        // If the batch is too large, we split the batch and send the split batches again. We do not decrement
        // the retry attempts in this case.
        log.warn("Got error produce response in correlation id {} on topic-partition {}, splitting and retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts(), formatErrMsg(response));
        if (transactionManager != null)
            transactionManager.removeInFlightBatch(batch);
        this.accumulator.splitAndReenqueue(batch);
        maybeRemoveAndDeallocateBatch(batch);
        this.sensors.recordBatchSplit();
    } else if (error != Errors.NONE) {
        if (canRetry(batch, response, now)) {
            log.warn("Got error produce response with correlation id {} on topic-partition {}, retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts() - 1, formatErrMsg(response));
            reenqueueBatch(batch, now);
        } else if (error == Errors.DUPLICATE_SEQUENCE_NUMBER) {
            // If we have received a duplicate sequence error, it means that the sequence number has advanced beyond
            // the sequence of the current batch, and we haven't retained batch metadata on the broker to return
            // the correct offset and timestamp.
            // 
            // The only thing we can do is to return success to the user and not return a valid offset and timestamp.
            completeBatch(batch, response);
        } else {
            // tell the user the result of their request. We only adjust sequence numbers if the batch didn't exhaust
            // its retries -- if it did, we don't know whether the sequence number was accepted or not, and
            // thus it is not safe to reassign the sequence.
            failBatch(batch, response, batch.attempts() < this.retries);
        }
        if (error.exception() instanceof InvalidMetadataException) {
            if (error.exception() instanceof UnknownTopicOrPartitionException) {
                log.warn("Received unknown topic or partition error in produce request on partition {}. The " + "topic-partition may not exist or the user may not have Describe access to it", batch.topicPartition);
            } else {
                log.warn("Received invalid metadata error in produce request on partition {} due to {}. Going " + "to request metadata update now", batch.topicPartition, error.exception(response.errorMessage).toString());
            }
            metadata.requestUpdate();
        }
    } else {
        completeBatch(batch, response);
    }
    // Unmute the completed partition.
    if (guaranteeMessageOrder)
        this.accumulator.unmutePartition(batch.topicPartition);
}

Example 3 with InvalidMetadataException

use of org.apache.kafka.common.errors.InvalidMetadataException in project apache-kafka-on-k8s by banzaicloud.

the class Sender method completeBatch.

/**
 * Complete or retry the given batch of records.
 *
 * @param batch The record batch
 * @param response The produce response
 * @param correlationId The correlation id for the request
 * @param now The current POSIX timestamp in milliseconds
 */
private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, long correlationId, long now) {
    Errors error = response.error;
    if (error == Errors.MESSAGE_TOO_LARGE && batch.recordCount > 1 && (batch.magic() >= RecordBatch.MAGIC_VALUE_V2 || batch.isCompressed())) {
        // If the batch is too large, we split the batch and send the split batches again. We do not decrement
        // the retry attempts in this case.
        log.warn("Got error produce response in correlation id {} on topic-partition {}, splitting and retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts(), error);
        if (transactionManager != null)
            transactionManager.removeInFlightBatch(batch);
        this.accumulator.splitAndReenqueue(batch);
        this.accumulator.deallocate(batch);
        this.sensors.recordBatchSplit();
    } else if (error != Errors.NONE) {
        if (canRetry(batch, response)) {
            log.warn("Got error produce response with correlation id {} on topic-partition {}, retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts() - 1, error);
            if (transactionManager == null) {
                reenqueueBatch(batch, now);
            } else if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) {
                // If idempotence is enabled only retry the request if the current producer id is the same as
                // the producer id of the batch.
                log.debug("Retrying batch to topic-partition {}. ProducerId: {}; Sequence number : {}", batch.topicPartition, batch.producerId(), batch.baseSequence());
                reenqueueBatch(batch, now);
            } else {
                failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false);
            }
        } else if (error == Errors.DUPLICATE_SEQUENCE_NUMBER) {
            // If we have received a duplicate sequence error, it means that the sequence number has advanced beyond
            // the sequence of the current batch, and we haven't retained batch metadata on the broker to return
            // the correct offset and timestamp.
            // 
            // The only thing we can do is to return success to the user and not return a valid offset and timestamp.
            completeBatch(batch, response);
        } else {
            final RuntimeException exception;
            if (error == Errors.TOPIC_AUTHORIZATION_FAILED)
                exception = new TopicAuthorizationException(batch.topicPartition.topic());
            else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED)
                exception = new ClusterAuthorizationException("The producer is not authorized to do idempotent sends");
            else
                exception = error.exception();
            // tell the user the result of their request. We only adjust sequence numbers if the batch didn't exhaust
            // its retries -- if it did, we don't know whether the sequence number was accepted or not, and
            // thus it is not safe to reassign the sequence.
            failBatch(batch, response, exception, batch.attempts() < this.retries);
        }
        if (error.exception() instanceof InvalidMetadataException) {
            if (error.exception() instanceof UnknownTopicOrPartitionException)
                log.warn("Received unknown topic or partition error in produce request on partition {}. The " + "topic/partition may not exist or the user may not have Describe access to it", batch.topicPartition);
            metadata.requestUpdate();
        }
    } else {
        completeBatch(batch, response);
    }
    // Unmute the completed partition.
    if (guaranteeMessageOrder)
        this.accumulator.unmutePartition(batch.topicPartition);
}