Examples with StreamsException org.apache.kafka.streams.errors.StreamsException used on opensource projects

Example 81 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class GlobalStreamThread method run.

@Override
public void run() {
    final StateConsumer stateConsumer = initialize();
    if (stateConsumer == null) {
        // during initialization, the caller thread would wait for the state consumer
        // to restore the global state store before transiting to RUNNING state and return;
        // if an error happens during the restoration process, the stateConsumer will be null
        // and in this case we will transit the state to PENDING_SHUTDOWN and DEAD immediately.
        // the exception will be thrown in the caller thread during start() function.
        setState(State.PENDING_SHUTDOWN);
        setState(State.DEAD);
        log.error("Error happened during initialization of the global state store; this thread has shutdown.");
        streamsMetrics.removeAllThreadLevelSensors(getName());
        streamsMetrics.removeAllThreadLevelMetrics(getName());
        return;
    }
    setState(RUNNING);
    boolean wipeStateStore = false;
    try {
        while (stillRunning()) {
            final long size = cacheSize.getAndSet(-1L);
            if (size != -1L) {
                cache.resize(size);
            }
            stateConsumer.pollAndUpdate();
        }
    } catch (final InvalidOffsetException recoverableException) {
        wipeStateStore = true;
        log.error("Updating global state failed due to inconsistent local state. Will attempt to clean up the local state. You can restart KafkaStreams to recover from this error.", recoverableException);
        final StreamsException e = new StreamsException("Updating global state failed. You can restart KafkaStreams to launch a new GlobalStreamThread to recover from this error.", recoverableException);
        this.streamsUncaughtExceptionHandler.accept(e);
    } catch (final Exception e) {
        log.error("Error happened while maintaining global state store. The streams application or client will now close to ERROR.", e);
        this.streamsUncaughtExceptionHandler.accept(e);
    } finally {
        // set the state to pending shutdown first as it may be called due to error;
        // its state may already be PENDING_SHUTDOWN so it will return false but we
        // intentionally do not check the returned flag
        setState(State.PENDING_SHUTDOWN);
        log.info("Shutting down");
        try {
            stateConsumer.close(wipeStateStore);
        } catch (final IOException e) {
            log.error("Failed to close state maintainer due to the following error:", e);
        }
        streamsMetrics.removeAllThreadLevelSensors(getName());
        streamsMetrics.removeAllThreadLevelMetrics(getName());
        setState(DEAD);
        log.info("Shutdown complete");
    }
}

Example 82 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class StreamsPartitionAssignor method populateClientStatesMap.

/**
 * Builds a map from client to state, and readies each ClientState for assignment by adding any missing prev tasks
 * and computing the per-task overall lag based on the fetched end offsets for each changelog.
 *
 * @param clientStates a map from each client to its state, including offset lags. Populated by this method.
 * @param clientMetadataMap a map from each client to its full metadata
 * @param taskForPartition map from topic partition to its corresponding task
 * @param changelogTopics object that manages changelog topics
 *
 * @return whether we were able to successfully fetch the changelog end offsets and compute each client's lag
 */
private boolean populateClientStatesMap(final Map<UUID, ClientState> clientStates, final Map<UUID, ClientMetadata> clientMetadataMap, final Map<TopicPartition, TaskId> taskForPartition, final ChangelogTopics changelogTopics) {
    boolean fetchEndOffsetsSuccessful;
    Map<TaskId, Long> allTaskEndOffsetSums;
    try {
        // Make the listOffsets request first so it can  fetch the offsets for non-source changelogs
        // asynchronously while we use the blocking Consumer#committed call to fetch source-changelog offsets
        final KafkaFuture<Map<TopicPartition, ListOffsetsResultInfo>> endOffsetsFuture = fetchEndOffsetsFuture(changelogTopics.preExistingNonSourceTopicBasedPartitions(), adminClient);
        final Map<TopicPartition, Long> sourceChangelogEndOffsets = fetchCommittedOffsets(changelogTopics.preExistingSourceTopicBasedPartitions(), mainConsumerSupplier.get());
        final Map<TopicPartition, ListOffsetsResultInfo> endOffsets = ClientUtils.getEndOffsets(endOffsetsFuture);
        allTaskEndOffsetSums = computeEndOffsetSumsByTask(endOffsets, sourceChangelogEndOffsets, changelogTopics);
        fetchEndOffsetsSuccessful = true;
    } catch (final StreamsException | TimeoutException e) {
        allTaskEndOffsetSums = changelogTopics.statefulTaskIds().stream().collect(Collectors.toMap(t -> t, t -> UNKNOWN_OFFSET_SUM));
        fetchEndOffsetsSuccessful = false;
    }
    for (final Map.Entry<UUID, ClientMetadata> entry : clientMetadataMap.entrySet()) {
        final UUID uuid = entry.getKey();
        final ClientState state = entry.getValue().state;
        state.initializePrevTasks(taskForPartition, taskManager.topologyMetadata().hasNamedTopologies());
        state.computeTaskLags(uuid, allTaskEndOffsetSums);
        clientStates.put(uuid, state);
    }
    return fetchEndOffsetsSuccessful;
}

Example 83 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class TaskManager method maybeWrapAndSetFirstException.

private void maybeWrapAndSetFirstException(final AtomicReference<RuntimeException> firstException, final RuntimeException exception, final TaskId taskId) {
    if (exception instanceof StreamsException) {
        ((StreamsException) exception).setTaskId(taskId);
        firstException.compareAndSet(null, exception);
    } else {
        firstException.compareAndSet(null, new StreamsException(exception, taskId));
    }
}

Example 84 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class TaskManager method handleAssignment.

/**
 * @throws TaskMigratedException if the task producer got fenced (EOS only)
 * @throws StreamsException fatal error while creating / initializing the task
 *
 * public for upgrade testing only
 */
public void handleAssignment(final Map<TaskId, Set<TopicPartition>> activeTasks, final Map<TaskId, Set<TopicPartition>> standbyTasks) {
    log.info("Handle new assignment with:\n" + "\tNew active tasks: {}\n" + "\tNew standby tasks: {}\n" + "\tExisting active tasks: {}\n" + "\tExisting standby tasks: {}", activeTasks.keySet(), standbyTasks.keySet(), activeTaskIds(), standbyTaskIds());
    topologyMetadata.addSubscribedTopicsFromAssignment(activeTasks.values().stream().flatMap(Collection::stream).collect(Collectors.toList()), logPrefix);
    final LinkedHashMap<TaskId, RuntimeException> taskCloseExceptions = new LinkedHashMap<>();
    final Map<TaskId, Set<TopicPartition>> activeTasksToCreate = new HashMap<>(activeTasks);
    final Map<TaskId, Set<TopicPartition>> standbyTasksToCreate = new HashMap<>(standbyTasks);
    final Comparator<Task> byId = Comparator.comparing(Task::id);
    final Set<Task> tasksToRecycle = new TreeSet<>(byId);
    final Set<Task> tasksToCloseClean = new TreeSet<>(byId);
    final Set<Task> tasksToCloseDirty = new TreeSet<>(byId);
    // first rectify all existing tasks
    for (final Task task : tasks.allTasks()) {
        if (activeTasks.containsKey(task.id()) && task.isActive()) {
            tasks.updateInputPartitionsAndResume(task, activeTasks.get(task.id()));
            activeTasksToCreate.remove(task.id());
        } else if (standbyTasks.containsKey(task.id()) && !task.isActive()) {
            tasks.updateInputPartitionsAndResume(task, standbyTasks.get(task.id()));
            standbyTasksToCreate.remove(task.id());
        } else if (activeTasks.containsKey(task.id()) || standbyTasks.containsKey(task.id())) {
            // check for tasks that were owned previously but have changed active/standby status
            tasksToRecycle.add(task);
        } else {
            tasksToCloseClean.add(task);
        }
    }
    // close and recycle those tasks
    handleCloseAndRecycle(tasksToRecycle, tasksToCloseClean, tasksToCloseDirty, activeTasksToCreate, standbyTasksToCreate, taskCloseExceptions);
    if (!taskCloseExceptions.isEmpty()) {
        log.error("Hit exceptions while closing / recycling tasks: {}", taskCloseExceptions);
        for (final Map.Entry<TaskId, RuntimeException> entry : taskCloseExceptions.entrySet()) {
            if (!(entry.getValue() instanceof TaskMigratedException)) {
                final TaskId taskId = entry.getKey();
                final RuntimeException exception = entry.getValue();
                if (exception instanceof StreamsException) {
                    ((StreamsException) exception).setTaskId(taskId);
                    throw exception;
                } else if (exception instanceof KafkaException) {
                    throw new StreamsException(exception, taskId);
                } else {
                    throw new StreamsException("Unexpected failure to close " + taskCloseExceptions.size() + " task(s) [" + taskCloseExceptions.keySet() + "]. " + "First unexpected exception (for task " + taskId + ") follows.", exception, taskId);
                }
            }
        }
        // If all exceptions are task-migrated, we would just throw the first one. No need to wrap with a
        // StreamsException since TaskMigrated is handled explicitly by the StreamThread
        final Map.Entry<TaskId, RuntimeException> first = taskCloseExceptions.entrySet().iterator().next();
        throw first.getValue();
    }
    tasks.handleNewAssignmentAndCreateTasks(activeTasksToCreate, standbyTasksToCreate, activeTasks.keySet(), standbyTasks.keySet());
}

Example 85 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class TaskExecutor method processTask.

private long processTask(final Task task, final int maxNumRecords, final long begin, final Time time) {
    int processed = 0;
    long now = begin;
    final long then = now;
    try {
        while (processed < maxNumRecords && task.process(now)) {
            task.clearTaskTimeout();
            processed++;
        }
        // TODO: enable regardless of whether using named topologies
        if (processed > 0 && hasNamedTopologies && processingMode != EXACTLY_ONCE_V2) {
            log.trace("Successfully processed task {}", task.id());
            tasks.addToSuccessfullyProcessed(task);
        }
    } catch (final TimeoutException timeoutException) {
        // TODO consolidate TimeoutException retries with general error handling
        task.maybeInitTaskTimeoutOrThrow(now, timeoutException);
        log.error(String.format("Could not complete processing records for %s due to the following exception; will move to next task and retry later", task.id()), timeoutException);
    } catch (final TaskMigratedException e) {
        log.info("Failed to process stream task {} since it got migrated to another thread already. " + "Will trigger a new rebalance and close all tasks as zombies together.", task.id());
        throw e;
    } catch (final StreamsException e) {
        log.error(String.format("Failed to process stream task %s due to the following error:", task.id()), e);
        e.setTaskId(task.id());
        throw e;
    } catch (final RuntimeException e) {
        log.error(String.format("Failed to process stream task %s due to the following error:", task.id()), e);
        throw new StreamsException(e, task.id());
    } finally {
        now = time.milliseconds();
        task.recordProcessBatchTime(now - then);
    }
    return processed;
}