Examples with StreamsException org.apache.kafka.streams.errors.StreamsException used on opensource projects

Example 31 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class StreamTask method process.

/**
 * Process one record.
 *
 * @return true if this method processes a record, false if it does not process a record.
 * @throws TaskMigratedException if the task producer got fenced (EOS only)
 */
@SuppressWarnings("unchecked")
public boolean process() {
    // get the next record to process
    final StampedRecord record = partitionGroup.nextRecord(recordInfo);
    // if there is no record to process, return immediately
    if (record == null) {
        return false;
    }
    try {
        // process the record by passing to the source node of the topology
        final ProcessorNode currNode = recordInfo.node();
        final TopicPartition partition = recordInfo.partition();
        log.trace("Start processing one record [{}]", record);
        updateProcessorContext(record, currNode);
        currNode.process(record.key(), record.value());
        log.trace("Completed processing one record [{}]", record);
        // update the consumed offset map after processing is done
        consumedOffsets.put(partition, record.offset());
        commitOffsetNeeded = true;
        // decreased to the threshold, we can then resume the consumption on this partition
        if (recordInfo.queue().size() == maxBufferedSize) {
            consumer.resume(singleton(partition));
        }
    } catch (final ProducerFencedException fatal) {
        throw new TaskMigratedException(this, fatal);
    } catch (final KafkaException e) {
        throw new StreamsException(format("Exception caught in process. taskId=%s, processor=%s, topic=%s, partition=%d, offset=%d", id(), processorContext.currentNode().name(), record.topic(), record.partition(), record.offset()), e);
    } finally {
        processorContext.setCurrentNode(null);
    }
    return true;
}

Example 32 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class StreamThread method maybeUpdateStandbyTasks.

private void maybeUpdateStandbyTasks(final long now) {
    if (state == State.RUNNING && taskManager.hasStandbyRunningTasks()) {
        if (processStandbyRecords) {
            if (!standbyRecords.isEmpty()) {
                final Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> remainingStandbyRecords = new HashMap<>();
                for (final Map.Entry<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> entry : standbyRecords.entrySet()) {
                    final TopicPartition partition = entry.getKey();
                    List<ConsumerRecord<byte[], byte[]>> remaining = entry.getValue();
                    if (remaining != null) {
                        final StandbyTask task = taskManager.standbyTask(partition);
                        if (task.isClosed()) {
                            log.warn("Standby task {} is already closed, probably because it got unexpectly migrated to another thread already. " + "Notifying the thread to trigger a new rebalance immediately.", task.id());
                            throw new TaskMigratedException(task);
                        }
                        remaining = task.update(partition, remaining);
                        if (remaining != null) {
                            remainingStandbyRecords.put(partition, remaining);
                        } else {
                            restoreConsumer.resume(singleton(partition));
                        }
                    }
                }
                standbyRecords = remainingStandbyRecords;
                log.debug("Updated standby tasks {} in {}ms", taskManager.standbyTaskIds(), time.milliseconds() - now);
            }
            processStandbyRecords = false;
        }
        try {
            final ConsumerRecords<byte[], byte[]> records = restoreConsumer.poll(0);
            if (!records.isEmpty()) {
                for (final TopicPartition partition : records.partitions()) {
                    final StandbyTask task = taskManager.standbyTask(partition);
                    if (task == null) {
                        throw new StreamsException(logPrefix + "Missing standby task for partition " + partition);
                    }
                    if (task.isClosed()) {
                        log.warn("Standby task {} is already closed, probably because it got unexpectedly migrated to another thread already. " + "Notifying the thread to trigger a new rebalance immediately.", task.id());
                        throw new TaskMigratedException(task);
                    }
                    final List<ConsumerRecord<byte[], byte[]>> remaining = task.update(partition, records.records(partition));
                    if (remaining != null) {
                        restoreConsumer.pause(singleton(partition));
                        standbyRecords.put(partition, remaining);
                    }
                }
            }
        } catch (final InvalidOffsetException recoverableException) {
            log.warn("Updating StandbyTasks failed. Deleting StandbyTasks stores to recreate from scratch.", recoverableException);
            final Set<TopicPartition> partitions = recoverableException.partitions();
            for (final TopicPartition partition : partitions) {
                final StandbyTask task = taskManager.standbyTask(partition);
                if (task.isClosed()) {
                    log.warn("Standby task {} is already closed, probably because it got unexpectly migrated to another thread already. " + "Notifying the thread to trigger a new rebalance immediately.", task.id());
                    throw new TaskMigratedException(task);
                }
                log.info("Reinitializing StandbyTask {}", task);
                task.reinitializeStateStoresForPartitions(recoverableException.partitions());
            }
            restoreConsumer.seekToBeginning(partitions);
        }
    }
}

Example 33 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class StreamThread method resetInvalidOffsets.

private void resetInvalidOffsets(final InvalidOffsetException e) {
    final Set<TopicPartition> partitions = e.partitions();
    final Set<String> loggedTopics = new HashSet<>();
    final Set<TopicPartition> seekToBeginning = new HashSet<>();
    final Set<TopicPartition> seekToEnd = new HashSet<>();
    for (final TopicPartition partition : partitions) {
        if (builder.earliestResetTopicsPattern().matcher(partition.topic()).matches()) {
            addToResetList(partition, seekToBeginning, "Setting topic '{}' to consume from {} offset", "earliest", loggedTopics);
        } else if (builder.latestResetTopicsPattern().matcher(partition.topic()).matches()) {
            addToResetList(partition, seekToEnd, "Setting topic '{}' to consume from {} offset", "latest", loggedTopics);
        } else {
            if (originalReset == null || (!originalReset.equals("earliest") && !originalReset.equals("latest"))) {
                final String errorMessage = "No valid committed offset found for input topic %s (partition %s) and no valid reset policy configured." + " You need to set configuration parameter \"auto.offset.reset\" or specify a topic specific reset " + "policy via StreamsBuilder#stream(..., Consumed.with(Topology.AutoOffsetReset)) or StreamsBuilder#table(..., Consumed.with(Topology.AutoOffsetReset))";
                throw new StreamsException(String.format(errorMessage, partition.topic(), partition.partition()), e);
            }
            if (originalReset.equals("earliest")) {
                addToResetList(partition, seekToBeginning, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "earliest", loggedTopics);
            } else if (originalReset.equals("latest")) {
                addToResetList(partition, seekToEnd, "No custom setting defined for topic '{}' using original config '{}' for offset reset", "latest", loggedTopics);
            }
        }
    }
    if (!seekToBeginning.isEmpty()) {
        consumer.seekToBeginning(seekToBeginning);
    }
    if (!seekToEnd.isEmpty()) {
        consumer.seekToEnd(seekToEnd);
    }
}

Example 34 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class TaskManager method addStreamTasks.

private void addStreamTasks(final Collection<TopicPartition> assignment) {
    if (assignedActiveTasks.isEmpty()) {
        return;
    }
    final Map<TaskId, Set<TopicPartition>> newTasks = new HashMap<>();
    // collect newly assigned tasks and reopen re-assigned tasks
    log.debug("Adding assigned tasks as active: {}", assignedActiveTasks);
    for (final Map.Entry<TaskId, Set<TopicPartition>> entry : assignedActiveTasks.entrySet()) {
        final TaskId taskId = entry.getKey();
        final Set<TopicPartition> partitions = entry.getValue();
        if (assignment.containsAll(partitions)) {
            try {
                if (!active.maybeResumeSuspendedTask(taskId, partitions)) {
                    newTasks.put(taskId, partitions);
                }
            } catch (final StreamsException e) {
                log.error("Failed to resume an active task {} due to the following error:", taskId, e);
                throw e;
            }
        } else {
            log.warn("Task {} owned partitions {} are not contained in the assignment {}", taskId, partitions, assignment);
        }
    }
    if (newTasks.isEmpty()) {
        return;
    }
    // CANNOT FIND RETRY AND BACKOFF LOGIC
    // create all newly assigned tasks (guard against race condition with other thread via backoff and retry)
    // -> other thread will call removeSuspendedTasks(); eventually
    log.trace("New active tasks to be created: {}", newTasks);
    for (final StreamTask task : taskCreator.createTasks(consumer, newTasks)) {
        active.addNewTask(task);
    }
}

Example 35 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class TaskManager method suspendTasksAndState.

/**
 * Similar to shutdownTasksAndState, however does not close the task managers, in the hope that
 * soon the tasks will be assigned again
 * @throws TaskMigratedException if the task producer got fenced (EOS only)
 */
void suspendTasksAndState() {
    log.debug("Suspending all active tasks {} and standby tasks {}", active.runningTaskIds(), standby.runningTaskIds());
    final AtomicReference<RuntimeException> firstException = new AtomicReference<>(null);
    firstException.compareAndSet(null, active.suspend());
    firstException.compareAndSet(null, standby.suspend());
    // remove the changelog partitions from restore consumer
    restoreConsumer.unsubscribe();
    final Exception exception = firstException.get();
    if (exception != null) {
        throw new StreamsException(logPrefix + "failed to suspend stream tasks", exception);
    }
}