Examples with TaskMigratedException org.apache.kafka.streams.errors.TaskMigratedException used on opensource projects

Example 11 with TaskMigratedException

use of org.apache.kafka.streams.errors.TaskMigratedException in project apache-kafka-on-k8s by banzaicloud.

the class TaskManager method suspendTasksAndState.

/**
 * Similar to shutdownTasksAndState, however does not close the task managers, in the hope that
 * soon the tasks will be assigned again
 * @throws TaskMigratedException if the task producer got fenced (EOS only)
 */
void suspendTasksAndState() {
    log.debug("Suspending all active tasks {} and standby tasks {}", active.runningTaskIds(), standby.runningTaskIds());
    final AtomicReference<RuntimeException> firstException = new AtomicReference<>(null);
    firstException.compareAndSet(null, active.suspend());
    firstException.compareAndSet(null, standby.suspend());
    // remove the changelog partitions from restore consumer
    restoreConsumer.unsubscribe();
    final Exception exception = firstException.get();
    if (exception != null) {
        throw new StreamsException(logPrefix + "failed to suspend stream tasks", exception);
    }
}

Example 12 with TaskMigratedException

use of org.apache.kafka.streams.errors.TaskMigratedException in project apache-kafka-on-k8s by banzaicloud.

the class StreamTask method commitOffsets.

/**
 * @throws TaskMigratedException if committing offsets failed (non-EOS)
 *                               or if the task producer got fenced (EOS)
 */
private void commitOffsets(final boolean startNewTransaction) {
    try {
        if (commitOffsetNeeded) {
            log.trace("Committing offsets");
            final Map<TopicPartition, OffsetAndMetadata> consumedOffsetsAndMetadata = new HashMap<>(consumedOffsets.size());
            for (final Map.Entry<TopicPartition, Long> entry : consumedOffsets.entrySet()) {
                final TopicPartition partition = entry.getKey();
                final long offset = entry.getValue() + 1;
                consumedOffsetsAndMetadata.put(partition, new OffsetAndMetadata(offset));
                stateMgr.putOffsetLimit(partition, offset);
            }
            if (eosEnabled) {
                producer.sendOffsetsToTransaction(consumedOffsetsAndMetadata, applicationId);
                producer.commitTransaction();
                transactionInFlight = false;
                if (startNewTransaction) {
                    transactionInFlight = true;
                    producer.beginTransaction();
                }
            } else {
                consumer.commitSync(consumedOffsetsAndMetadata);
            }
            commitOffsetNeeded = false;
        } else if (eosEnabled && !startNewTransaction && transactionInFlight) {
            // need to make sure to commit txn for suspend case
            producer.commitTransaction();
            transactionInFlight = false;
        }
    } catch (final CommitFailedException | ProducerFencedException fatal) {
        throw new TaskMigratedException(this, fatal);
    }
}

Example 13 with TaskMigratedException

use of org.apache.kafka.streams.errors.TaskMigratedException in project apache-kafka-on-k8s by banzaicloud.

the class StreamThreadTest method shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWhileProcessing.

@Test
public void shouldCloseTaskAsZombieAndRemoveFromActiveTasksIfProducerWasFencedWhileProcessing() throws Exception {
    internalTopologyBuilder.addSource(null, "source", null, null, null, topic1);
    internalTopologyBuilder.addSink("sink", "dummyTopic", null, null, null, "source");
    final StreamThread thread = createStreamThread(clientId, new StreamsConfig(configProps(true)), true);
    final MockConsumer<byte[], byte[]> consumer = clientSupplier.consumer;
    consumer.updatePartitions(topic1, Collections.singletonList(new PartitionInfo(topic1, 1, null, null, null)));
    thread.setState(StreamThread.State.RUNNING);
    thread.rebalanceListener.onPartitionsRevoked(null);
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    final List<TopicPartition> assignedPartitions = new ArrayList<>();
    // assign single partition
    assignedPartitions.add(t1p1);
    activeTasks.put(task1, Collections.singleton(t1p1));
    thread.taskManager().setAssignmentMetadata(activeTasks, Collections.<TaskId, Set<TopicPartition>>emptyMap());
    final MockConsumer<byte[], byte[]> mockConsumer = (MockConsumer<byte[], byte[]>) thread.consumer;
    mockConsumer.assign(assignedPartitions);
    mockConsumer.updateBeginningOffsets(Collections.singletonMap(t1p1, 0L));
    thread.rebalanceListener.onPartitionsAssigned(assignedPartitions);
    thread.runOnce(-1);
    assertThat(thread.tasks().size(), equalTo(1));
    final MockProducer producer = clientSupplier.producers.get(0);
    // change consumer subscription from "pattern" to "manual" to be able to call .addRecords()
    consumer.updateBeginningOffsets(Collections.singletonMap(assignedPartitions.iterator().next(), 0L));
    consumer.unsubscribe();
    consumer.assign(new HashSet<>(assignedPartitions));
    consumer.addRecord(new ConsumerRecord<>(topic1, 1, 0, new byte[0], new byte[0]));
    mockTime.sleep(config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG) + 1);
    thread.runOnce(-1);
    assertThat(producer.history().size(), equalTo(1));
    assertFalse(producer.transactionCommitted());
    mockTime.sleep(config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG) + 1L);
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return producer.commitCount() == 1;
        }
    }, "StreamsThread did not commit transaction.");
    producer.fenceProducer();
    mockTime.sleep(config.getLong(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG) + 1L);
    consumer.addRecord(new ConsumerRecord<>(topic1, 1, 0, new byte[0], new byte[0]));
    try {
        thread.runOnce(-1);
        fail("Should have thrown TaskMigratedException");
    } catch (final TaskMigratedException expected) {
    /* ignore */
    }
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return thread.tasks().isEmpty();
        }
    }, "StreamsThread did not remove fenced zombie task.");
    assertThat(producer.commitCount(), equalTo(1L));
}

Example 14 with TaskMigratedException

use of org.apache.kafka.streams.errors.TaskMigratedException in project apache-kafka-on-k8s by banzaicloud.

the class AssignedStreamsTasksTest method shouldCloseTaskOnSuspendIfTaskMigratedException.

@Test
public void shouldCloseTaskOnSuspendIfTaskMigratedException() {
    mockTaskInitialization();
    t1.suspend();
    EasyMock.expectLastCall().andThrow(new TaskMigratedException(t1));
    t1.close(false, true);
    EasyMock.expectLastCall();
    EasyMock.replay(t1);
    assertThat(suspendTask(), nullValue());
    assertTrue(assignedTasks.previousTaskIds().isEmpty());
    EasyMock.verify(t1);
}

Example 15 with TaskMigratedException

use of org.apache.kafka.streams.errors.TaskMigratedException in project apache-kafka-on-k8s by banzaicloud.

the class AssignedStreamsTasksTest method shouldCloseTaskOnMaybePunctuateStreamTimeIfTaskMigratedException.

@Test
public void shouldCloseTaskOnMaybePunctuateStreamTimeIfTaskMigratedException() {
    mockTaskInitialization();
    t1.maybePunctuateStreamTime();
    EasyMock.expectLastCall().andThrow(new TaskMigratedException(t1));
    t1.close(false, true);
    EasyMock.expectLastCall();
    EasyMock.replay(t1);
    addAndInitTask();
    try {
        assignedTasks.punctuate();
        fail("Should have thrown TaskMigratedException.");
    } catch (final TaskMigratedException expected) {
    /* ignore */
    }
    assertThat(assignedTasks.runningTaskIds(), equalTo(Collections.EMPTY_SET));
    EasyMock.verify(t1);
}