Examples with RetriableException org.apache.kafka.connect.errors.RetriableException used on opensource projects

Example 1 with RetriableException

use of org.apache.kafka.connect.errors.RetriableException in project kafka by apache.

the class WorkerSinkTaskTest method testPollRedelivery.

@Test
public void testPollRedelivery() throws Exception {
    expectInitializeTask();
    expectPollInitialAssignment();
    // If a retriable exception is thrown, we should redeliver the same batch, pausing the consumer in the meantime
    expectConsumerPoll(1);
    expectConversionAndTransformation(1);
    Capture<Collection<SinkRecord>> records = EasyMock.newCapture(CaptureType.ALL);
    sinkTask.put(EasyMock.capture(records));
    EasyMock.expectLastCall().andThrow(new RetriableException("retry"));
    // Pause
    HashSet<TopicPartition> partitions = new HashSet<>(asList(TOPIC_PARTITION, TOPIC_PARTITION2));
    EasyMock.expect(consumer.assignment()).andReturn(partitions);
    consumer.pause(partitions);
    PowerMock.expectLastCall();
    // Retry delivery should succeed
    expectConsumerPoll(0);
    sinkTask.put(EasyMock.capture(records));
    EasyMock.expectLastCall();
    // And unpause
    EasyMock.expect(consumer.assignment()).andReturn(partitions);
    consumer.resume(singleton(TOPIC_PARTITION));
    PowerMock.expectLastCall();
    consumer.resume(singleton(TOPIC_PARTITION2));
    PowerMock.expectLastCall();
    PowerMock.replayAll();
    workerTask.initialize(TASK_CONFIG);
    workerTask.initializeAndStart();
    workerTask.iteration();
    workerTask.iteration();
    workerTask.iteration();
    PowerMock.verifyAll();
}

Example 2 with RetriableException

use of org.apache.kafka.connect.errors.RetriableException in project kafka by apache.

the class WorkerSinkTask method deliverMessages.

private void deliverMessages() {
    // Finally, deliver this batch to the sink
    try {
        // Since we reuse the messageBatch buffer, ensure we give the task its own copy
        task.put(new ArrayList<>(messageBatch));
        for (SinkRecord record : messageBatch) currentOffsets.put(new TopicPartition(record.topic(), record.kafkaPartition()), new OffsetAndMetadata(record.kafkaOffset() + 1));
        messageBatch.clear();
        // the task had not explicitly paused
        if (pausedForRedelivery) {
            if (!shouldPause())
                resumeAll();
            pausedForRedelivery = false;
        }
    } catch (RetriableException e) {
        log.error("RetriableException from SinkTask {}:", id, e);
        // If we're retrying a previous batch, make sure we've paused all topic partitions so we don't get new data,
        // but will still be able to poll in order to handle user-requested timeouts, keep group membership, etc.
        pausedForRedelivery = true;
        pauseAll();
    // Let this exit normally, the batch will be reprocessed on the next loop.
    } catch (Throwable t) {
        log.error("Task {} threw an uncaught and unrecoverable exception", id, t);
        log.error("Task is being killed and will not recover until manually restarted");
        throw new ConnectException("Exiting WorkerSinkTask due to unrecoverable exception.");
    }
}