Examples with TaskCorruptedException org.apache.kafka.streams.errors.TaskCorruptedException used on opensource projects

Example 1 with TaskCorruptedException

use of org.apache.kafka.streams.errors.TaskCorruptedException in project kafka by apache.

the class StreamThreadTest method shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath.

@Test
@SuppressWarnings("unchecked")
public void shouldCatchHandleCorruptionOnTaskCorruptedExceptionPath() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    consumer.subscribe((Collection<String>) anyObject(), anyObject());
    EasyMock.expectLastCall().anyTimes();
    consumer.unsubscribe();
    EasyMock.expectLastCall().anyTimes();
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    EasyMock.replay(consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 0);
    final TaskId taskId2 = new TaskId(0, 2);
    final Set<TaskId> corruptedTasks = singleton(taskId1);
    expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task1.id()).andReturn(taskId1).anyTimes();
    expect(task2.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task2.id()).andReturn(taskId2).anyTimes();
    expect(taskManager.handleCorruption(corruptedTasks)).andReturn(true);
    EasyMock.replay(task1, task2, taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, config, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, HANDLER, null) {

        @Override
        void runOnce() {
            setState(State.PENDING_SHUTDOWN);
            throw new TaskCorruptedException(corruptedTasks);
        }
    }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    thread.run();
    verify(taskManager);
}

Example 2 with TaskCorruptedException

use of org.apache.kafka.streams.errors.TaskCorruptedException in project kafka by apache.

the class StreamThreadTest method shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath.

@Test
@SuppressWarnings("unchecked")
public void shouldCatchTaskMigratedExceptionOnOnTaskCorruptedExceptionPath() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    consumer.subscribe((Collection<String>) anyObject(), anyObject());
    EasyMock.expectLastCall().anyTimes();
    consumer.unsubscribe();
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 0);
    final TaskId taskId2 = new TaskId(0, 2);
    final Set<TaskId> corruptedTasks = singleton(taskId1);
    expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task1.id()).andReturn(taskId1).anyTimes();
    expect(task2.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task2.id()).andReturn(taskId2).anyTimes();
    taskManager.handleCorruption(corruptedTasks);
    expectLastCall().andThrow(new TaskMigratedException("Task migrated", new RuntimeException("non-corrupted task migrated")));
    taskManager.handleLostAll();
    expectLastCall();
    EasyMock.replay(task1, task2, taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, config, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, HANDLER, null) {

        @Override
        void runOnce() {
            setState(State.PENDING_SHUTDOWN);
            throw new TaskCorruptedException(corruptedTasks);
        }
    }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    thread.setState(StreamThread.State.STARTING);
    thread.runLoop();
    verify(taskManager);
}

Example 3 with TaskCorruptedException

use of org.apache.kafka.streams.errors.TaskCorruptedException in project kafka by apache.

the class StreamThreadTest method shouldEnforceRebalanceWhenTaskCorruptedExceptionIsThrownForAnActiveTask.

@Test
@SuppressWarnings("unchecked")
public void shouldEnforceRebalanceWhenTaskCorruptedExceptionIsThrownForAnActiveTask() {
    final TaskManager taskManager = EasyMock.createNiceMock(TaskManager.class);
    expect(taskManager.producerClientIds()).andStubReturn(Collections.emptySet());
    final Consumer<byte[], byte[]> consumer = mock(Consumer.class);
    final ConsumerGroupMetadata consumerGroupMetadata = mock(ConsumerGroupMetadata.class);
    expect(consumer.groupMetadata()).andStubReturn(consumerGroupMetadata);
    expect(consumerGroupMetadata.groupInstanceId()).andReturn(Optional.empty());
    consumer.subscribe((Collection<String>) anyObject(), anyObject());
    EasyMock.expectLastCall().anyTimes();
    consumer.unsubscribe();
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(consumerGroupMetadata);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    final TaskId taskId1 = new TaskId(0, 0);
    final TaskId taskId2 = new TaskId(0, 2);
    final Set<TaskId> corruptedTasks = singleton(taskId1);
    expect(task1.state()).andReturn(Task.State.RUNNING).anyTimes();
    expect(task1.id()).andReturn(taskId1).anyTimes();
    expect(task2.state()).andReturn(Task.State.CREATED).anyTimes();
    expect(task2.id()).andReturn(taskId2).anyTimes();
    expect(taskManager.handleCorruption(corruptedTasks)).andReturn(true);
    consumer.enforceRebalance();
    expectLastCall();
    EasyMock.replay(task1, task2, taskManager, consumer);
    final StreamsMetricsImpl streamsMetrics = new StreamsMetricsImpl(metrics, CLIENT_ID, StreamsConfig.METRICS_LATEST, mockTime);
    final TopologyMetadata topologyMetadata = new TopologyMetadata(internalTopologyBuilder, config);
    topologyMetadata.buildAndRewriteTopology();
    final StreamThread thread = new StreamThread(mockTime, eosEnabledConfig, null, consumer, consumer, null, null, taskManager, streamsMetrics, topologyMetadata, CLIENT_ID, new LogContext(""), new AtomicInteger(), new AtomicLong(Long.MAX_VALUE), new LinkedList<>(), null, HANDLER, null) {

        @Override
        void runOnce() {
            setState(State.PENDING_SHUTDOWN);
            throw new TaskCorruptedException(corruptedTasks);
        }
    }.updateThreadMetadata(getSharedAdminClientId(CLIENT_ID));
    thread.setState(StreamThread.State.STARTING);
    thread.runLoop();
    verify(taskManager);
    verify(consumer);
}

Example 4 with TaskCorruptedException

use of org.apache.kafka.streams.errors.TaskCorruptedException in project kafka by apache.

the class TaskManagerTest method shouldNotFailForTimeoutExceptionOnCommitWithEosAlpha.

@Test
public void shouldNotFailForTimeoutExceptionOnCommitWithEosAlpha() {
    setUpTaskManager(ProcessingMode.EXACTLY_ONCE_ALPHA);
    final StreamsProducer producer = mock(StreamsProducer.class);
    expect(activeTaskCreator.streamsProducerForTask(anyObject(TaskId.class))).andReturn(producer).andReturn(producer).andReturn(producer);
    final Map<TopicPartition, OffsetAndMetadata> offsetsT00 = singletonMap(t1p0, new OffsetAndMetadata(0L, null));
    final Map<TopicPartition, OffsetAndMetadata> offsetsT01 = singletonMap(t1p1, new OffsetAndMetadata(1L, null));
    producer.commitTransaction(offsetsT00, null);
    expectLastCall().andThrow(new TimeoutException("KABOOM!"));
    producer.commitTransaction(offsetsT00, null);
    expectLastCall();
    producer.commitTransaction(offsetsT01, null);
    expectLastCall();
    producer.commitTransaction(offsetsT01, null);
    expectLastCall();
    final StateMachineTask task00 = new StateMachineTask(taskId00, taskId00Partitions, true);
    task00.setCommittableOffsetsAndMetadata(offsetsT00);
    final StateMachineTask task01 = new StateMachineTask(taskId01, taskId01Partitions, true);
    task01.setCommittableOffsetsAndMetadata(offsetsT01);
    final StateMachineTask task02 = new StateMachineTask(taskId02, taskId02Partitions, true);
    expect(consumer.groupMetadata()).andStubReturn(null);
    replay(producer, activeTaskCreator, consumer);
    task00.setCommitNeeded();
    task01.setCommitNeeded();
    final TaskCorruptedException exception = assertThrows(TaskCorruptedException.class, () -> taskManager.commit(mkSet(task00, task01, task02)));
    assertThat(exception.corruptedTasks(), equalTo(Collections.singleton(taskId00)));
}

Example 5 with TaskCorruptedException

use of org.apache.kafka.streams.errors.TaskCorruptedException in project kafka by apache.

the class RecordCollectorImpl method recordSendError.

private void recordSendError(final String topic, final Exception exception, final ProducerRecord<byte[], byte[]> serializedRecord) {
    String errorMessage = String.format(SEND_EXCEPTION_MESSAGE, topic, taskId, exception.toString());
    if (isFatalException(exception)) {
        errorMessage += "\nWritten offsets would not be recorded and no more records would be sent since this is a fatal error.";
        sendException.set(new StreamsException(errorMessage, exception));
    } else if (exception instanceof ProducerFencedException || exception instanceof InvalidProducerEpochException || exception instanceof OutOfOrderSequenceException) {
        errorMessage += "\nWritten offsets would not be recorded and no more records would be sent since the producer is fenced, " + "indicating the task may be migrated out";
        sendException.set(new TaskMigratedException(errorMessage, exception));
    } else {
        if (exception instanceof RetriableException) {
            errorMessage += "\nThe broker is either slow or in bad state (like not having enough replicas) in responding the request, " + "or the connection to broker was interrupted sending the request or receiving the response. " + "\nConsider overwriting `max.block.ms` and /or " + "`delivery.timeout.ms` to a larger value to wait longer for such scenarios and avoid timeout errors";
            sendException.set(new TaskCorruptedException(Collections.singleton(taskId)));
        } else {
            if (productionExceptionHandler.handle(serializedRecord, exception) == ProductionExceptionHandlerResponse.FAIL) {
                errorMessage += "\nException handler choose to FAIL the processing, no more records would be sent.";
                sendException.set(new StreamsException(errorMessage, exception));
            } else {
                errorMessage += "\nException handler choose to CONTINUE processing in spite of this error but written offsets would not be recorded.";
                droppedRecordsSensor.record();
            }
        }
    }
    log.error(errorMessage, exception);
}