Examples with StreamsException org.apache.kafka.streams.errors.StreamsException used on opensource projects

Example 36 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class InternalTopicManager method makeReady.

/**
 * Prepares a set of given internal topics.
 *
 * If a topic does not exist creates a new topic.
 * If a topic with the correct number of partitions exists ignores it.
 * If a topic exists already but has different number of partitions we fail and throw exception requesting user to reset the app before restarting again.
 */
public void makeReady(final Map<String, InternalTopicConfig> topics) {
    final Map<String, Integer> existingTopicPartitions = getNumPartitions(topics.keySet());
    final Set<InternalTopicConfig> topicsToBeCreated = validateTopicPartitions(topics.values(), existingTopicPartitions);
    if (topicsToBeCreated.size() > 0) {
        final Set<NewTopic> newTopics = new HashSet<>();
        for (final InternalTopicConfig internalTopicConfig : topicsToBeCreated) {
            final Map<String, String> topicConfig = internalTopicConfig.getProperties(defaultTopicConfigs, windowChangeLogAdditionalRetention);
            log.debug("Going to create topic {} with {} partitions and config {}.", internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), topicConfig);
            newTopics.add(new NewTopic(internalTopicConfig.name(), internalTopicConfig.numberOfPartitions(), replicationFactor).configs(topicConfig));
        }
        int remainingRetries = retries;
        boolean retry;
        do {
            retry = false;
            final CreateTopicsResult createTopicsResult = adminClient.createTopics(newTopics);
            final Set<String> createTopicNames = new HashSet<>();
            for (final Map.Entry<String, KafkaFuture<Void>> createTopicResult : createTopicsResult.values().entrySet()) {
                try {
                    createTopicResult.getValue().get();
                    createTopicNames.add(createTopicResult.getKey());
                } catch (final ExecutionException couldNotCreateTopic) {
                    final Throwable cause = couldNotCreateTopic.getCause();
                    final String topicName = createTopicResult.getKey();
                    if (cause instanceof TimeoutException) {
                        retry = true;
                        log.debug("Could not get number of partitions for topic {} due to timeout. " + "Will try again (remaining retries {}).", topicName, remainingRetries - 1);
                    } else if (cause instanceof TopicExistsException) {
                        createTopicNames.add(createTopicResult.getKey());
                        log.info(String.format("Topic %s exist already: %s", topicName, couldNotCreateTopic.getMessage()));
                    } else {
                        throw new StreamsException(String.format("Could not create topic %s.", topicName), couldNotCreateTopic);
                    }
                } catch (final InterruptedException fatalException) {
                    Thread.currentThread().interrupt();
                    log.error(INTERRUPTED_ERROR_MESSAGE, fatalException);
                    throw new IllegalStateException(INTERRUPTED_ERROR_MESSAGE, fatalException);
                }
            }
            if (retry) {
                final Iterator<NewTopic> it = newTopics.iterator();
                while (it.hasNext()) {
                    if (createTopicNames.contains(it.next().name())) {
                        it.remove();
                    }
                }
                continue;
            }
            return;
        } while (remainingRetries-- > 0);
        final String timeoutAndRetryError = "Could not create topics. " + "This can happen if the Kafka cluster is temporary not available. " + "You can increase admin client config `retries` to be resilient against this error.";
        log.error(timeoutAndRetryError);
        throw new StreamsException(timeoutAndRetryError);
    }
}

Example 37 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class AbstractStateManager method reinitializeStateStoresForPartitions.

public void reinitializeStateStoresForPartitions(final Logger log, final Map<String, StateStore> stateStores, final Map<String, String> storeToChangelogTopic, final Collection<TopicPartition> partitions, final InternalProcessorContext processorContext) {
    final Map<String, String> changelogTopicToStore = inverseOneToOneMap(storeToChangelogTopic);
    final Set<String> storeToBeReinitialized = new HashSet<>();
    final Map<String, StateStore> storesCopy = new HashMap<>(stateStores);
    for (final TopicPartition topicPartition : partitions) {
        checkpointableOffsets.remove(topicPartition);
        storeToBeReinitialized.add(changelogTopicToStore.get(topicPartition.topic()));
    }
    try {
        checkpoint.write(checkpointableOffsets);
    } catch (final IOException fatalException) {
        log.error("Failed to write offset checkpoint file to {} while re-initializing {}: {}", checkpoint, stateStores, fatalException);
        throw new StreamsException("Failed to reinitialize global store.", fatalException);
    }
    for (final Map.Entry<String, StateStore> entry : storesCopy.entrySet()) {
        final StateStore stateStore = entry.getValue();
        final String storeName = stateStore.name();
        if (storeToBeReinitialized.contains(storeName)) {
            try {
                stateStore.close();
            } catch (final RuntimeException ignoreAndSwallow) {
            /* ignore */
            }
            processorContext.uninitialize();
            stateStores.remove(entry.getKey());
            // store directory as <taskDir>/<storeName> but nests it with an intermediate <taskDir>/rocksdb/<storeName>
            try {
                Utils.delete(new File(baseDir + File.separator + "rocksdb" + File.separator + storeName));
            } catch (final IOException fatalException) {
                log.error("Failed to reinitialize store {}.", storeName, fatalException);
                throw new StreamsException(String.format("Failed to reinitialize store %s.", storeName), fatalException);
            }
            try {
                Utils.delete(new File(baseDir + File.separator + storeName));
            } catch (final IOException fatalException) {
                log.error("Failed to reinitialize store {}.", storeName, fatalException);
                throw new StreamsException(String.format("Failed to reinitialize store %s.", storeName), fatalException);
            }
            stateStore.init(processorContext, stateStore);
        }
    }
}

Example 38 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class GlobalStateManagerImpl method topicPartitionsForStore.

private List<TopicPartition> topicPartitionsForStore(final StateStore store) {
    final String sourceTopic = topology.storeToChangelogTopic().get(store.name());
    List<PartitionInfo> partitionInfos;
    int attempts = 0;
    while (true) {
        try {
            partitionInfos = globalConsumer.partitionsFor(sourceTopic);
            break;
        } catch (final TimeoutException retryableException) {
            if (++attempts > retries) {
                log.error("Failed to get partitions for topic {} after {} retry attempts due to timeout. " + "The broker may be transiently unavailable at the moment. " + "You can increase the number of retries via configuration parameter `retries`.", sourceTopic, retries, retryableException);
                throw new StreamsException(String.format("Failed to get partitions for topic %s after %d retry attempts due to timeout. " + "The broker may be transiently unavailable at the moment. " + "You can increase the number of retries via configuration parameter `retries`.", sourceTopic, retries), retryableException);
            }
            log.debug("Failed to get partitions for topic {} due to timeout. The broker may be transiently unavailable at the moment. " + "Backing off for {} ms to retry (attempt {} of {})", sourceTopic, retryBackoffMs, attempts, retries, retryableException);
            Utils.sleep(retryBackoffMs);
        }
    }
    if (partitionInfos == null || partitionInfos.isEmpty()) {
        throw new StreamsException(String.format("There are no partitions available for topic %s when initializing global store %s", sourceTopic, store.name()));
    }
    final List<TopicPartition> topicPartitions = new ArrayList<>();
    for (PartitionInfo partition : partitionInfos) {
        topicPartitions.add(new TopicPartition(partition.topic(), partition.partition()));
    }
    return topicPartitions;
}

Example 39 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class GlobalStateManagerImpl method initialize.

@Override
public Set<String> initialize() {
    try {
        if (!stateDirectory.lockGlobalState()) {
            throw new LockException(String.format("Failed to lock the global state directory: %s", baseDir));
        }
    } catch (IOException e) {
        throw new LockException(String.format("Failed to lock the global state directory: %s", baseDir));
    }
    try {
        this.checkpointableOffsets.putAll(checkpoint.read());
    } catch (IOException e) {
        try {
            stateDirectory.unlockGlobalState();
        } catch (IOException e1) {
            log.error("Failed to unlock the global state directory", e);
        }
        throw new StreamsException("Failed to read checkpoints for global state globalStores", e);
    }
    final List<StateStore> stateStores = topology.globalStateStores();
    for (final StateStore stateStore : stateStores) {
        globalStoreNames.add(stateStore.name());
        stateStore.init(processorContext, stateStore);
    }
    return Collections.unmodifiableSet(globalStoreNames);
}

Example 40 with StreamsException

use of org.apache.kafka.streams.errors.StreamsException in project apache-kafka-on-k8s by banzaicloud.

the class StoreChangelogReader method initialize.

private void initialize() {
    if (!restoreConsumer.subscription().isEmpty()) {
        throw new StreamsException("Restore consumer should not be subscribed to any topics (" + restoreConsumer.subscription() + ")");
    }
    // first refresh the changelog partition information from brokers, since initialize is only called when
    // the needsInitializing map is not empty, meaning we do not know the metadata for some of them yet
    refreshChangelogInfo();
    Map<TopicPartition, StateRestorer> initializable = new HashMap<>();
    for (Map.Entry<TopicPartition, StateRestorer> entry : needsInitializing.entrySet()) {
        final TopicPartition topicPartition = entry.getKey();
        if (hasPartition(topicPartition)) {
            initializable.put(entry.getKey(), entry.getValue());
        }
    }
    // where we already have all of the data
    try {
        endOffsets.putAll(restoreConsumer.endOffsets(initializable.keySet()));
    } catch (final TimeoutException e) {
        // if timeout exception gets thrown we just give up this time and retry in the next run loop
        log.debug("Could not fetch end offset for {}; will fall back to partition by partition fetching", initializable);
        return;
    }
    final Iterator<TopicPartition> iter = initializable.keySet().iterator();
    while (iter.hasNext()) {
        final TopicPartition topicPartition = iter.next();
        final Long endOffset = endOffsets.get(topicPartition);
        // we add this check just in case
        if (endOffset != null) {
            final StateRestorer restorer = needsInitializing.get(topicPartition);
            if (restorer.checkpoint() >= endOffset) {
                restorer.setRestoredOffset(restorer.checkpoint());
                iter.remove();
            } else if (restorer.offsetLimit() == 0 || endOffset == 0) {
                restorer.setRestoredOffset(0);
                iter.remove();
            } else {
                restorer.setEndingOffset(endOffset);
            }
            needsInitializing.remove(topicPartition);
        } else {
            log.info("End offset cannot be found form the returned metadata; removing this partition from the current run loop");
            iter.remove();
        }
    }
    // set up restorer for those initializable
    if (!initializable.isEmpty()) {
        startRestoration(initializable);
    }
}