Examples with StreamsException org.apache.kafka.streams.errors.StreamsException used on opensource projects

Example 71 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class InternalTopicManager method cleanUpCreatedTopics.

private void cleanUpCreatedTopics(final Set<String> topicsToCleanUp) {
    log.info("Starting to clean up internal topics {}.", topicsToCleanUp);
    final long now = time.milliseconds();
    final long deadline = now + retryTimeoutMs;
    final Set<String> topicsStillToCleanup = new HashSet<>(topicsToCleanUp);
    while (!topicsStillToCleanup.isEmpty()) {
        log.info("Going to cleanup internal topics: " + topicsStillToCleanup);
        final DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(topicsStillToCleanup);
        final Map<String, KafkaFuture<Void>> deleteResultForTopic = deleteTopicsResult.topicNameValues();
        while (!deleteResultForTopic.isEmpty()) {
            for (final String topicName : new HashSet<>(topicsStillToCleanup)) {
                if (!deleteResultForTopic.containsKey(topicName)) {
                    throw new IllegalStateException("Delete topic results do not contain internal topic " + topicName + " to clean up. " + BUG_ERROR_MESSAGE);
                }
                final KafkaFuture<Void> deleteResult = deleteResultForTopic.get(topicName);
                if (deleteResult.isDone()) {
                    try {
                        deleteResult.get();
                        topicsStillToCleanup.remove(topicName);
                    } catch (final ExecutionException executionException) {
                        final Throwable cause = executionException.getCause();
                        if (cause instanceof UnknownTopicOrPartitionException) {
                            log.info("Internal topic {} to clean up is missing", topicName);
                        } else if (cause instanceof LeaderNotAvailableException) {
                            log.info("The leader of internal topic {} to clean up is not available.", topicName);
                        } else if (cause instanceof TimeoutException) {
                            log.info("Cleaning up internal topic {} timed out.", topicName);
                        } else {
                            log.error("Unexpected error during cleanup of internal topics: ", cause);
                            throw new StreamsException(String.format("Could not clean up internal topics %s, because during the cleanup " + "of topic %s the following error occurred: ", topicsStillToCleanup, topicName), cause);
                        }
                    } catch (final InterruptedException interruptedException) {
                        throw new InterruptException(interruptedException);
                    } finally {
                        deleteResultForTopic.remove(topicName);
                    }
                }
            }
            maybeThrowTimeoutException(Collections.singletonList(topicsStillToCleanup), deadline, String.format("Could not cleanup internal topics within %d milliseconds. This can happen if the " + "Kafka cluster is temporarily not available or the broker did not complete topic creation " + "before the cleanup. The following internal topics could not be cleaned up: %s", retryTimeoutMs, topicsStillToCleanup));
            if (!deleteResultForTopic.isEmpty()) {
                Utils.sleep(100);
            }
        }
        maybeSleep(Collections.singletonList(topicsStillToCleanup), deadline, "validated");
    }
    log.info("Completed cleanup of internal topics {}.", topicsToCleanUp);
}

Example 72 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class InternalTopicManager method doValidateTopic.

private <V> void doValidateTopic(final ValidationResult validationResult, final Map<String, KafkaFuture<V>> futuresForTopic, final Map<String, InternalTopicConfig> topicsConfigs, final Set<String> topicsStillToValidate, final BiConsumer<InternalTopicConfig, V> validator) {
    for (final String topicName : new HashSet<>(topicsStillToValidate)) {
        if (!futuresForTopic.containsKey(topicName)) {
            throw new IllegalStateException("Description results do not contain topics to validate. " + BUG_ERROR_MESSAGE);
        }
        final KafkaFuture<V> future = futuresForTopic.get(topicName);
        if (future.isDone()) {
            try {
                final V brokerSideTopicConfig = future.get();
                final InternalTopicConfig streamsSideTopicConfig = topicsConfigs.get(topicName);
                validator.accept(streamsSideTopicConfig, brokerSideTopicConfig);
                topicsStillToValidate.remove(topicName);
            } catch (final ExecutionException executionException) {
                final Throwable cause = executionException.getCause();
                if (cause instanceof UnknownTopicOrPartitionException) {
                    log.info("Internal topic {} is missing", topicName);
                    validationResult.addMissingTopic(topicName);
                    topicsStillToValidate.remove(topicName);
                } else if (cause instanceof LeaderNotAvailableException) {
                    log.info("The leader of internal topic {} is not available.", topicName);
                } else if (cause instanceof TimeoutException) {
                    log.info("Retrieving data for internal topic {} timed out.", topicName);
                } else {
                    log.error("Unexpected error during internal topic validation: ", cause);
                    throw new StreamsException(String.format("Could not validate internal topic %s for the following reason: ", topicName), cause);
                }
            } catch (final InterruptedException interruptedException) {
                throw new InterruptException(interruptedException);
            } finally {
                futuresForTopic.remove(topicName);
            }
        }
    }
}

Example 73 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class InternalTopicManager method getNumPartitions.

/**
 * Try to get the number of partitions for the given topics; return the number of partitions for topics that already exists.
 *
 * Topics that were not able to get its description will simply not be returned
 */
// visible for testing
protected Map<String, Integer> getNumPartitions(final Set<String> topics, final Set<String> tempUnknownTopics) {
    log.debug("Trying to check if topics {} have been created with expected number of partitions.", topics);
    final DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(topics);
    final Map<String, KafkaFuture<TopicDescription>> futures = describeTopicsResult.topicNameValues();
    final Map<String, Integer> existedTopicPartition = new HashMap<>();
    for (final Map.Entry<String, KafkaFuture<TopicDescription>> topicFuture : futures.entrySet()) {
        final String topicName = topicFuture.getKey();
        try {
            final TopicDescription topicDescription = topicFuture.getValue().get();
            existedTopicPartition.put(topicName, topicDescription.partitions().size());
        } catch (final InterruptedException fatalException) {
            // this should not happen; if it ever happens it indicate a bug
            Thread.currentThread().interrupt();
            log.error(INTERRUPTED_ERROR_MESSAGE, fatalException);
            throw new IllegalStateException(INTERRUPTED_ERROR_MESSAGE, fatalException);
        } catch (final ExecutionException couldNotDescribeTopicException) {
            final Throwable cause = couldNotDescribeTopicException.getCause();
            if (cause instanceof UnknownTopicOrPartitionException) {
                // This topic didn't exist, proceed to try to create it
                log.debug("Topic {} is unknown or not found, hence not existed yet.\n" + "Error message was: {}", topicName, cause.toString());
            } else if (cause instanceof LeaderNotAvailableException) {
                tempUnknownTopics.add(topicName);
                log.debug("The leader of topic {} is not available.\n" + "Error message was: {}", topicName, cause.toString());
            } else {
                log.error("Unexpected error during topic description for {}.\n" + "Error message was: {}", topicName, cause.toString());
                throw new StreamsException(String.format("Could not create topic %s.", topicName), cause);
            }
        } catch (final TimeoutException retriableException) {
            tempUnknownTopics.add(topicName);
            log.debug("Describing topic {} (to get number of partitions) timed out.\n" + "Error message was: {}", topicName, retriableException.toString());
        }
    }
    return existedTopicPartition;
}

Example 74 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class ProcessorNode method init.

public void init(final InternalProcessorContext<KOut, VOut> context) {
    if (!closed)
        throw new IllegalStateException("The processor is not closed");
    try {
        threadId = Thread.currentThread().getName();
        internalProcessorContext = context;
        if (processor != null) {
            processor.init(context);
        }
    } catch (final Exception e) {
        throw new StreamsException(String.format("failed to initialize processor %s", name), e);
    }
    // revived tasks could re-initialize the topology,
    // in which case we should reset the flag
    closed = false;
}

Example 75 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project kafka by apache.

the class ProcessorStateManager method flush.

/**
 * @throws TaskMigratedException recoverable error sending changelog records that would cause the task to be removed
 * @throws StreamsException fatal error when flushing the state store, for example sending changelog records failed
 *                          or flushing state store get IO errors; such error should cause the thread to die
 */
@Override
public void flush() {
    RuntimeException firstException = null;
    // attempting to flush the stores
    if (!stores.isEmpty()) {
        log.debug("Flushing all stores registered in the state manager: {}", stores);
        for (final StateStoreMetadata metadata : stores.values()) {
            final StateStore store = metadata.stateStore;
            log.trace("Flushing store {}", store.name());
            try {
                store.flush();
            } catch (final RuntimeException exception) {
                if (firstException == null) {
                    // do NOT wrap the error if it is actually caused by Streams itself
                    if (exception instanceof StreamsException)
                        firstException = exception;
                    else
                        firstException = new ProcessorStateException(format("%sFailed to flush state store %s", logPrefix, store.name()), exception);
                }
                log.error("Failed to flush state store {}: ", store.name(), exception);
            }
        }
    }
    if (firstException != null) {
        throw firstException;
    }
}