Examples with Source org.apache.kafka.streams.TopologyDescription.Source used on opensource projects

Example 1 with Source

use of org.apache.kafka.streams.TopologyDescription.Source in project ksql by confluentinc.

the class RestTestExecutor method waitForPersistentQueriesToProcessInputs.

/**
 * This method looks at all of the source topics that feed into all of the subtopologies and waits
 * for the consumer group associated with each application to reach the end offsets for those
 * topics.  This effectively ensures that nothing is lagging when it completes successfully.
 * This should ensure that any materialized state has been built up correctly and is ready for
 * pull queries.
 */
private void waitForPersistentQueriesToProcessInputs() {
    // First wait for the queries to be in the RUNNING state
    boolean allRunning = false;
    final long queryRunningThreshold = System.currentTimeMillis() + MAX_QUERY_RUNNING_CHECK.toMillis();
    while (System.currentTimeMillis() < queryRunningThreshold) {
        boolean notReady = false;
        for (PersistentQueryMetadata persistentQueryMetadata : engine.getPersistentQueries()) {
            if (persistentQueryMetadata.getState() != State.RUNNING) {
                notReady = true;
            }
        }
        if (notReady) {
            threadYield();
        } else {
            allRunning = true;
            break;
        }
    }
    if (!allRunning) {
        throw new AssertionError("Timed out while trying to wait for queries to begin running");
    }
    // Collect all application ids
    List<String> queryApplicationIds = engine.getPersistentQueries().stream().map(QueryMetadata::getQueryApplicationId).collect(Collectors.toList());
    // Collect all possible source topic names for each application id
    Map<String, Set<String>> possibleTopicNamesByAppId = engine.getPersistentQueries().stream().collect(Collectors.toMap(QueryMetadata::getQueryApplicationId, m -> {
        Set<String> topics = getSourceTopics(m);
        Set<String> possibleInternalNames = topics.stream().map(t -> m.getQueryApplicationId() + "-" + t).collect(Collectors.toSet());
        Set<String> all = new HashSet<>();
        all.addAll(topics);
        all.addAll(possibleInternalNames);
        return all;
    }));
    final Set<String> possibleTopicNames = possibleTopicNamesByAppId.values().stream().flatMap(Collection::stream).collect(Collectors.toSet());
    // Every topic is either internal or not, so we expect to match exactly half of them.
    int expectedTopics = possibleTopicNames.size() / 2;
    // Find the intersection of possible topic names and real topic names, and wait until the
    // expected number are all there
    final Set<String> topics = new HashSet<>();
    boolean foundTopics = false;
    final long topicThreshold = System.currentTimeMillis() + MAX_TOPIC_NAME_LOOKUP.toMillis();
    while (System.currentTimeMillis() < topicThreshold) {
        Set<String> expectedNames = kafkaCluster.getTopics();
        expectedNames.retainAll(possibleTopicNames);
        if (expectedNames.size() == expectedTopics) {
            foundTopics = true;
            topics.addAll(expectedNames);
            break;
        }
    }
    if (!foundTopics) {
        throw new AssertionError("Timed out while trying to find topics");
    }
    // Only retain topic names which are known to exist.
    Map<String, Set<String>> topicNamesByAppId = possibleTopicNamesByAppId.entrySet().stream().collect(Collectors.toMap(Entry::getKey, e -> {
        e.getValue().retainAll(topics);
        return e.getValue();
    }));
    Map<String, Integer> partitionCount = kafkaCluster.getPartitionCount(topics);
    Map<String, List<TopicPartition>> topicPartitionsByAppId = queryApplicationIds.stream().collect(Collectors.toMap(appId -> appId, appId -> {
        final List<TopicPartition> allTopicPartitions = new ArrayList<>();
        for (String topic : topicNamesByAppId.get(appId)) {
            for (int i = 0; i < partitionCount.get(topic); i++) {
                final TopicPartition tp = new TopicPartition(topic, i);
                allTopicPartitions.add(tp);
            }
        }
        return allTopicPartitions;
    }));
    final long threshold = System.currentTimeMillis() + MAX_STATIC_WARM_UP.toMillis();
    mainloop: while (System.currentTimeMillis() < threshold) {
        for (String queryApplicationId : queryApplicationIds) {
            final List<TopicPartition> topicPartitions = topicPartitionsByAppId.get(queryApplicationId);
            Map<TopicPartition, Long> currentOffsets = kafkaCluster.getConsumerGroupOffset(queryApplicationId);
            Map<TopicPartition, Long> endOffsets = kafkaCluster.getEndOffsets(topicPartitions, // Since we're doing At Least Once, we can do read uncommitted.
            IsolationLevel.READ_COMMITTED);
            for (final TopicPartition tp : topicPartitions) {
                if (!currentOffsets.containsKey(tp) && endOffsets.get(tp) > 0) {
                    LOG.info("Haven't committed offsets yet for " + tp + " end offset " + endOffsets.get(tp));
                    threadYield();
                    continue mainloop;
                }
            }
            for (final Map.Entry<TopicPartition, Long> entry : currentOffsets.entrySet()) {
                final TopicPartition tp = entry.getKey();
                final long currentOffset = entry.getValue();
                final long endOffset = endOffsets.get(tp);
                if (currentOffset < endOffset) {
                    LOG.info("Offsets are not caught up current: " + currentOffsets + " end: " + endOffsets);
                    threadYield();
                    continue mainloop;
                }
            }
        }
        LOG.info("Offsets are all up to date");
        return;
    }
    LOG.info("Timed out waiting for correct response");
    throw new AssertionError("Timed out while trying to wait for offsets");
}

Example 2 with Source

use of org.apache.kafka.streams.TopologyDescription.Source in project ksql by confluentinc.

the class RestTestExecutor method getSourceTopics.

private Set<String> getSourceTopics(final PersistentQueryMetadata persistentQueryMetadata) {
    Set<String> topics = new HashSet<>();
    for (final Subtopology subtopology : persistentQueryMetadata.getTopology().describe().subtopologies()) {
        for (final TopologyDescription.Node node : subtopology.nodes()) {
            if (node instanceof Source) {
                final Source source = (Source) node;
                Preconditions.checkNotNull(source.topicSet(), "Expecting topic set, not regex");
                topics.addAll(source.topicSet());
            }
        }
    }
    return topics;
}

Example 3 with Source

use of org.apache.kafka.streams.TopologyDescription.Source in project ksql by confluentinc.

the class KsLocator method findSubtopologySourceTopicSuffixes.

/**
 * For the particular state store, this finds the subtopology which contains that store, and
 * then finds the input topics for the subtopology by finding the source nodes. These topics are
 * then used for checking against all the metadata for the state store and used to find the
 * active and standby hosts for the topics. Without doing this check, incorrect assignments could
 * be chosen since different subtopologies can be run by different hosts.
 */
private Set<String> findSubtopologySourceTopicSuffixes() {
    for (final Subtopology subtopology : topology.describe().subtopologies()) {
        boolean containsStateStore = false;
        for (final TopologyDescription.Node node : subtopology.nodes()) {
            if (node instanceof Processor) {
                final Processor processor = (Processor) node;
                if (processor.stores().contains(storeName)) {
                    containsStateStore = true;
                }
            }
        }
        if (!containsStateStore) {
            continue;
        }
        for (final TopologyDescription.Node node : subtopology.nodes()) {
            if (node instanceof Source) {
                final Source source = (Source) node;
                Preconditions.checkNotNull(source.topicSet(), "Expecting topic set, not regex");
                return source.topicSet();
            }
        }
        throw new IllegalStateException("Failed to find source with topics");
    }
    throw new IllegalStateException("Failed to find state store " + storeName);
}