Examples with HostInfo org.apache.kafka.streams.state.HostInfo used on opensource projects

Example 56 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project ksql by confluentinc.

the class KsLocator method getMetadataForKeys.

/**
 * Gets the Metadata when looking up a list of keys.  This is used when the set of keys are known.
 * @param keys The non-empty set of keys to lookup metadata for
 * @param filterPartitions The partitions to limit lookups to, if non empty. Partitions which
 *                         exist by are not listed here are omitted. If empty, no filtering is
 *                         done.
 * @return The metadata associated with the keys
 */
private List<PartitionMetadata> getMetadataForKeys(final List<KsqlKey> keys, final Set<Integer> filterPartitions) {
    // Maintain request order for reproducibility by using a LinkedHashMap, even though it's
    // not a guarantee of the API.
    final Map<Integer, KeyQueryMetadata> metadataByPartition = new LinkedHashMap<>();
    final Map<Integer, Set<KsqlKey>> keysByPartition = new HashMap<>();
    for (KsqlKey key : keys) {
        final KeyQueryMetadata metadata = getKeyQueryMetadata(key);
        // Fail fast if Streams not ready. Let client handle it
        if (metadata.equals(KeyQueryMetadata.NOT_AVAILABLE)) {
            LOG.debug("KeyQueryMetadata not available for state store '{}' and key {}", storeName, key);
            throw new MaterializationException(String.format("Materialized data for key %s is not available yet. " + "Please try again later.", key));
        }
        LOG.debug("Handling pull query for key {} in partition {} of state store {}.", key, metadata.partition(), storeName);
        if (filterPartitions.size() > 0 && !filterPartitions.contains(metadata.partition())) {
            LOG.debug("Ignoring key {} in partition {} because parition is not included in lookup.", key, metadata.partition());
            continue;
        }
        keysByPartition.computeIfAbsent(metadata.partition(), k -> new LinkedHashSet<>());
        keysByPartition.get(metadata.partition()).add(key);
        metadataByPartition.putIfAbsent(metadata.partition(), metadata);
    }
    return metadataByPartition.values().stream().map(metadata -> {
        final HostInfo activeHost = metadata.activeHost();
        final Set<HostInfo> standByHosts = metadata.standbyHosts();
        return new PartitionMetadata(activeHost, standByHosts, metadata.partition(), Optional.of(keysByPartition.get(metadata.partition())));
    }).collect(Collectors.toList());
}

Example 57 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project ksql by confluentinc.

the class KsLocator method getMetadataForAllPartitions.

/**
 * Gets the metadata for all partitions associated with the state store.
 * @param filterPartitions The partitions to limit lookups to, if non empty. Partitions which
 *                         exist by are not listed here are omitted. If empty, no filtering is
 *                         done.
 * @return The metadata associated with all partitions
 */
private List<PartitionMetadata> getMetadataForAllPartitions(final Set<Integer> filterPartitions, final Optional<Set<KsqlKey>> keys) {
    // It's important that we consider only the source topics for the subtopology that contains the
    // state store. Otherwise, we'll be given the wrong partition -> host mappings.
    // The underlying state store has a number of partitions that is the MAX of the number of
    // partitions of all source topics of the subtopology.  Since partition X of all source topics
    // of the particular subtopology will map to the same host, we can collect partition -> host
    // for these topics to find the locations of each partition of the state store.
    final Set<String> sourceTopicSuffixes = findSubtopologySourceTopicSuffixes();
    final Map<Integer, HostInfo> activeHostByPartition = new HashMap<>();
    final Map<Integer, Set<HostInfo>> standbyHostsByPartition = new HashMap<>();
    final Collection<StreamsMetadata> streamsMetadataCollection = getStreamsMetadata();
    for (final StreamsMetadata streamsMetadata : streamsMetadataCollection) {
        streamsMetadata.topicPartitions().forEach(tp -> {
            if (sourceTopicSuffixes.stream().anyMatch(suffix -> tp.topic().endsWith(suffix))) {
                activeHostByPartition.compute(tp.partition(), (partition, hostInfo) -> {
                    if (hostInfo != null && !streamsMetadata.hostInfo().equals(hostInfo)) {
                        throw new IllegalStateException("Should only be one active host per partition");
                    }
                    return streamsMetadata.hostInfo();
                });
            }
        });
        streamsMetadata.standbyTopicPartitions().forEach(tp -> {
            // with multiple per partition, we can't easy sanity check.
            if (sourceTopicSuffixes.stream().anyMatch(suffix -> tp.topic().endsWith(suffix))) {
                standbyHostsByPartition.computeIfAbsent(tp.partition(), p -> new HashSet<>());
                standbyHostsByPartition.get(tp.partition()).add(streamsMetadata.hostInfo());
            }
        });
    }
    final Set<Integer> partitions = Streams.concat(activeHostByPartition.keySet().stream(), standbyHostsByPartition.keySet().stream()).collect(Collectors.toSet());
    final List<PartitionMetadata> metadataList = new ArrayList<>();
    for (Integer partition : partitions) {
        if (filterPartitions.size() > 0 && !filterPartitions.contains(partition)) {
            LOG.debug("Ignoring partition {} because partition is not included in lookup.", partition);
            continue;
        }
        final HostInfo activeHost = activeHostByPartition.getOrDefault(partition, UNKNOWN_HOST);
        final Set<HostInfo> standbyHosts = standbyHostsByPartition.getOrDefault(partition, Collections.emptySet());
        metadataList.add(new PartitionMetadata(activeHost, standbyHosts, partition, keys));
    }
    return metadataList;
}