Examples with OutOfOrderSequenceException org.apache.kafka.common.errors.OutOfOrderSequenceException used on opensource projects

Example 1 with OutOfOrderSequenceException

use of org.apache.kafka.common.errors.OutOfOrderSequenceException in project apache-kafka-on-k8s by banzaicloud.

the class TransactionalMessageCopier method main.

public static void main(String[] args) throws IOException {
    Namespace parsedArgs = argParser().parseArgsOrFail(args);
    Integer numMessagesPerTransaction = parsedArgs.getInt("messagesPerTransaction");
    final String transactionalId = parsedArgs.getString("transactionalId");
    final String outputTopic = parsedArgs.getString("outputTopic");
    String consumerGroup = parsedArgs.getString("consumerGroup");
    TopicPartition inputPartition = new TopicPartition(parsedArgs.getString("inputTopic"), parsedArgs.getInt("inputPartition"));
    final KafkaProducer<String, String> producer = createProducer(parsedArgs);
    final KafkaConsumer<String, String> consumer = createConsumer(parsedArgs);
    consumer.assign(singleton(inputPartition));
    long maxMessages = parsedArgs.getInt("maxMessages") == -1 ? Long.MAX_VALUE : parsedArgs.getInt("maxMessages");
    maxMessages = Math.min(messagesRemaining(consumer, inputPartition), maxMessages);
    final boolean enableRandomAborts = parsedArgs.getBoolean("enableRandomAborts");
    producer.initTransactions();
    final AtomicBoolean isShuttingDown = new AtomicBoolean(false);
    final AtomicLong remainingMessages = new AtomicLong(maxMessages);
    final AtomicLong numMessagesProcessed = new AtomicLong(0);
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            isShuttingDown.set(true);
            // Flush any remaining messages
            producer.close();
            synchronized (consumer) {
                consumer.close();
            }
            System.out.println(shutDownString(numMessagesProcessed.get(), remainingMessages.get(), transactionalId));
        }
    });
    try {
        Random random = new Random();
        while (0 < remainingMessages.get()) {
            System.out.println(statusAsJson(numMessagesProcessed.get(), remainingMessages.get(), transactionalId));
            if (isShuttingDown.get())
                break;
            int messagesInCurrentTransaction = 0;
            long numMessagesForNextTransaction = Math.min(numMessagesPerTransaction, remainingMessages.get());
            try {
                producer.beginTransaction();
                while (messagesInCurrentTransaction < numMessagesForNextTransaction) {
                    ConsumerRecords<String, String> records = consumer.poll(200L);
                    for (ConsumerRecord<String, String> record : records) {
                        producer.send(producerRecordFromConsumerRecord(outputTopic, record));
                        messagesInCurrentTransaction++;
                    }
                }
                producer.sendOffsetsToTransaction(consumerPositions(consumer), consumerGroup);
                if (enableRandomAborts && random.nextInt() % 3 == 0) {
                    throw new KafkaException("Aborting transaction");
                } else {
                    producer.commitTransaction();
                    remainingMessages.set(maxMessages - numMessagesProcessed.addAndGet(messagesInCurrentTransaction));
                }
            } catch (ProducerFencedException | OutOfOrderSequenceException e) {
                // We cannot recover from these errors, so just rethrow them and let the process fail
                throw e;
            } catch (KafkaException e) {
                producer.abortTransaction();
                resetToLastCommittedPositions(consumer);
            }
        }
    } finally {
        producer.close();
        synchronized (consumer) {
            consumer.close();
        }
    }
    System.exit(0);
}

Example 2 with OutOfOrderSequenceException

use of org.apache.kafka.common.errors.OutOfOrderSequenceException in project apache-kafka-on-k8s by banzaicloud.

the class Sender method completeBatch.

/**
 * Complete or retry the given batch of records.
 *
 * @param batch The record batch
 * @param response The produce response
 * @param correlationId The correlation id for the request
 * @param now The current POSIX timestamp in milliseconds
 */
private void completeBatch(ProducerBatch batch, ProduceResponse.PartitionResponse response, long correlationId, long now) {
    Errors error = response.error;
    if (error == Errors.MESSAGE_TOO_LARGE && batch.recordCount > 1 && (batch.magic() >= RecordBatch.MAGIC_VALUE_V2 || batch.isCompressed())) {
        // If the batch is too large, we split the batch and send the split batches again. We do not decrement
        // the retry attempts in this case.
        log.warn("Got error produce response in correlation id {} on topic-partition {}, splitting and retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts(), error);
        if (transactionManager != null)
            transactionManager.removeInFlightBatch(batch);
        this.accumulator.splitAndReenqueue(batch);
        this.accumulator.deallocate(batch);
        this.sensors.recordBatchSplit();
    } else if (error != Errors.NONE) {
        if (canRetry(batch, response)) {
            log.warn("Got error produce response with correlation id {} on topic-partition {}, retrying ({} attempts left). Error: {}", correlationId, batch.topicPartition, this.retries - batch.attempts() - 1, error);
            if (transactionManager == null) {
                reenqueueBatch(batch, now);
            } else if (transactionManager.hasProducerIdAndEpoch(batch.producerId(), batch.producerEpoch())) {
                // If idempotence is enabled only retry the request if the current producer id is the same as
                // the producer id of the batch.
                log.debug("Retrying batch to topic-partition {}. ProducerId: {}; Sequence number : {}", batch.topicPartition, batch.producerId(), batch.baseSequence());
                reenqueueBatch(batch, now);
            } else {
                failBatch(batch, response, new OutOfOrderSequenceException("Attempted to retry sending a " + "batch but the producer id changed from " + batch.producerId() + " to " + transactionManager.producerIdAndEpoch().producerId + " in the mean time. This batch will be dropped."), false);
            }
        } else if (error == Errors.DUPLICATE_SEQUENCE_NUMBER) {
            // If we have received a duplicate sequence error, it means that the sequence number has advanced beyond
            // the sequence of the current batch, and we haven't retained batch metadata on the broker to return
            // the correct offset and timestamp.
            // 
            // The only thing we can do is to return success to the user and not return a valid offset and timestamp.
            completeBatch(batch, response);
        } else {
            final RuntimeException exception;
            if (error == Errors.TOPIC_AUTHORIZATION_FAILED)
                exception = new TopicAuthorizationException(batch.topicPartition.topic());
            else if (error == Errors.CLUSTER_AUTHORIZATION_FAILED)
                exception = new ClusterAuthorizationException("The producer is not authorized to do idempotent sends");
            else
                exception = error.exception();
            // tell the user the result of their request. We only adjust sequence numbers if the batch didn't exhaust
            // its retries -- if it did, we don't know whether the sequence number was accepted or not, and
            // thus it is not safe to reassign the sequence.
            failBatch(batch, response, exception, batch.attempts() < this.retries);
        }
        if (error.exception() instanceof InvalidMetadataException) {
            if (error.exception() instanceof UnknownTopicOrPartitionException)
                log.warn("Received unknown topic or partition error in produce request on partition {}. The " + "topic/partition may not exist or the user may not have Describe access to it", batch.topicPartition);
            metadata.requestUpdate();
        }
    } else {
        completeBatch(batch, response);
    }
    // Unmute the completed partition.
    if (guaranteeMessageOrder)
        this.accumulator.unmutePartition(batch.topicPartition);
}

Example 3 with OutOfOrderSequenceException

use of org.apache.kafka.common.errors.OutOfOrderSequenceException in project apache-kafka-on-k8s by banzaicloud.

the class SenderTest method testBatchesDrainedWithOldProducerIdShouldFailWithOutOfOrderSequenceOnSubsequentRetry.

@Test
public void testBatchesDrainedWithOldProducerIdShouldFailWithOutOfOrderSequenceOnSubsequentRetry() throws Exception {
    final long producerId = 343434L;
    TransactionManager transactionManager = new TransactionManager();
    transactionManager.setProducerIdAndEpoch(new ProducerIdAndEpoch(producerId, (short) 0));
    setupWithTransactionState(transactionManager);
    client.setNode(new Node(1, "localhost", 33343));
    int maxRetries = 10;
    Metrics m = new Metrics();
    SenderMetricsRegistry senderMetrics = new SenderMetricsRegistry(m);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, true, MAX_REQUEST_SIZE, ACKS_ALL, maxRetries, senderMetrics, time, REQUEST_TIMEOUT, 50, transactionManager, apiVersions);
    Future<RecordMetadata> failedResponse = accumulator.append(tp0, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    Future<RecordMetadata> successfulResponse = accumulator.append(tp1, time.milliseconds(), "key".getBytes(), "value".getBytes(), null, null, MAX_BLOCK_TIMEOUT).future;
    // connect.
    sender.run(time.milliseconds());
    // send.
    sender.run(time.milliseconds());
    assertEquals(1, client.inFlightRequestCount());
    Map<TopicPartition, OffsetAndError> responses = new LinkedHashMap<>();
    responses.put(tp1, new OffsetAndError(-1, Errors.NOT_LEADER_FOR_PARTITION));
    responses.put(tp0, new OffsetAndError(-1, Errors.OUT_OF_ORDER_SEQUENCE_NUMBER));
    client.respond(produceResponse(responses));
    sender.run(time.milliseconds());
    assertTrue(failedResponse.isDone());
    assertFalse("Expected transaction state to be reset upon receiving an OutOfOrderSequenceException", transactionManager.hasProducerId());
    prepareAndReceiveInitProducerId(producerId + 1, Errors.NONE);
    assertEquals(producerId + 1, transactionManager.producerIdAndEpoch().producerId);
    // send request to tp1 with the old producerId
    sender.run(time.milliseconds());
    assertFalse(successfulResponse.isDone());
    // The response comes back with a retriable error.
    client.respond(produceResponse(tp1, 0, Errors.NOT_LEADER_FOR_PARTITION, -1));
    sender.run(time.milliseconds());
    assertTrue(successfulResponse.isDone());
    // exception.
    try {
        successfulResponse.get();
        fail("Should have raised an OutOfOrderSequenceException");
    } catch (Exception e) {
        assertTrue(e.getCause() instanceof OutOfOrderSequenceException);
    }
}