Examples with StreamsMetadataImpl org.apache.kafka.streams.state.internals.StreamsMetadataImpl used on opensource projects

Example 6 with StreamsMetadataImpl

use of org.apache.kafka.streams.state.internals.StreamsMetadataImpl in project kafka by apache.

the class StreamsMetadataState method rebuildMetadataForNamedTopologies.

private List<StreamsMetadata> rebuildMetadataForNamedTopologies(final Map<HostInfo, Set<TopicPartition>> activePartitionHostMap, final Map<HostInfo, Set<TopicPartition>> standbyPartitionHostMap) {
    final List<StreamsMetadata> rebuiltMetadata = new ArrayList<>();
    Stream.concat(activePartitionHostMap.keySet().stream(), standbyPartitionHostMap.keySet().stream()).distinct().sorted(Comparator.comparing(HostInfo::host).thenComparingInt(HostInfo::port)).forEach(hostInfo -> {
        for (final String topologyName : topologyMetadata.namedTopologiesView()) {
            final Map<String, List<String>> storeToSourceTopics = topologyMetadata.stateStoreNameToSourceTopicsForTopology(topologyName);
            final Set<TopicPartition> activePartitionsOnHost = new HashSet<>();
            final Set<String> activeStoresOnHost = new HashSet<>();
            if (activePartitionHostMap.containsKey(hostInfo)) {
                // filter out partitions for topics that are not connected to this topology
                activePartitionsOnHost.addAll(activePartitionHostMap.get(hostInfo).stream().filter(tp -> topologyMetadata.fullSourceTopicNamesForTopology(topologyName).contains(tp.topic())).collect(Collectors.toSet()));
                activeStoresOnHost.addAll(getStoresOnHost(storeToSourceTopics, activePartitionsOnHost));
            }
            // TODO KAFKA-13281: when we add support for global stores with named topologies we will
            // need to add the global stores to the activeStoresOnHost set
            final Set<TopicPartition> standbyPartitionsOnHost = new HashSet<>();
            final Set<String> standbyStoresOnHost = new HashSet<>();
            if (standbyPartitionHostMap.containsKey(hostInfo)) {
                standbyPartitionsOnHost.addAll(standbyPartitionHostMap.get(hostInfo).stream().filter(tp -> topologyMetadata.fullSourceTopicNamesForTopology(topologyName).contains(tp.topic())).collect(Collectors.toSet()));
                standbyStoresOnHost.addAll(getStoresOnHost(storeToSourceTopics, standbyPartitionsOnHost));
            }
            if (!(activeStoresOnHost.isEmpty() && activePartitionsOnHost.isEmpty() && standbyStoresOnHost.isEmpty() && standbyPartitionsOnHost.isEmpty())) {
                final StreamsMetadata metadata = new StreamsMetadataImpl(hostInfo, activeStoresOnHost, activePartitionsOnHost, standbyStoresOnHost, standbyPartitionsOnHost, topologyName);
                rebuiltMetadata.add(metadata);
                if (hostInfo.equals(thisHost)) {
                    localMetadata.set(metadata);
                }
            } else {
                log.debug("Host {} has no tasks for topology {} at the moment, this metadata will not be built", hostInfo, topologyName);
            }
        }
        // Construct metadata across all topologies on this host for the `localMetadata` field
        final Map<String, List<String>> storeToSourceTopics = topologyMetadata.stateStoreNameToSourceTopics();
        final Set<TopicPartition> localActivePartitions = activePartitionHostMap.get(thisHost);
        final Set<TopicPartition> localStandbyPartitions = standbyPartitionHostMap.get(thisHost);
        localMetadata.set(new StreamsMetadataImpl(thisHost, getStoresOnHost(storeToSourceTopics, localActivePartitions), localActivePartitions, getStoresOnHost(storeToSourceTopics, localStandbyPartitions), localStandbyPartitions));
    });
    return rebuiltMetadata;
}

Example 7 with StreamsMetadataImpl

use of org.apache.kafka.streams.state.internals.StreamsMetadataImpl in project kafka by apache.

the class StreamsMetadataTest method shouldNotBeEqualIfDifferInStandByStores.

@Test
public void shouldNotBeEqualIfDifferInStandByStores() {
    final StreamsMetadata differStandByStores = new StreamsMetadataImpl(HOST_INFO, STATE_STORE_NAMES, TOPIC_PARTITIONS, mkSet("store1"), STANDBY_TOPIC_PARTITIONS);
    assertThat(streamsMetadata, not(equalTo(differStandByStores)));
    assertThat(streamsMetadata.hashCode(), not(equalTo(differStandByStores.hashCode())));
}

Example 8 with StreamsMetadataImpl

use of org.apache.kafka.streams.state.internals.StreamsMetadataImpl in project kafka by apache.

the class StreamsMetadataTest method shouldNotBeEqualIfDifferInTopicPartitions.

@Test
public void shouldNotBeEqualIfDifferInTopicPartitions() {
    final StreamsMetadata differTopicPartitions = new StreamsMetadataImpl(HOST_INFO, STATE_STORE_NAMES, mkSet(TP_0), STAND_BY_STORE_NAMES, STANDBY_TOPIC_PARTITIONS);
    assertThat(streamsMetadata, not(equalTo(differTopicPartitions)));
    assertThat(streamsMetadata.hashCode(), not(equalTo(differTopicPartitions.hashCode())));
}

Example 9 with StreamsMetadataImpl

use of org.apache.kafka.streams.state.internals.StreamsMetadataImpl in project kafka by apache.

the class StreamsMetadataTest method shouldBeEqualsIfSameObject.

@Test
public void shouldBeEqualsIfSameObject() {
    final StreamsMetadata same = new StreamsMetadataImpl(HOST_INFO, STATE_STORE_NAMES, TOPIC_PARTITIONS, STAND_BY_STORE_NAMES, STANDBY_TOPIC_PARTITIONS);
    assertThat(streamsMetadata, equalTo(same));
    assertThat(streamsMetadata.hashCode(), equalTo(same.hashCode()));
}

Example 10 with StreamsMetadataImpl

use of org.apache.kafka.streams.state.internals.StreamsMetadataImpl in project kafka by apache.

the class StreamsMetadataState method rebuildMetadata.

private void rebuildMetadata(final Map<HostInfo, Set<TopicPartition>> activePartitionHostMap, final Map<HostInfo, Set<TopicPartition>> standbyPartitionHostMap) {
    if (activePartitionHostMap.isEmpty() && standbyPartitionHostMap.isEmpty()) {
        allMetadata = Collections.emptyList();
        localMetadata.set(new StreamsMetadataImpl(thisHost, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet()));
        return;
    }
    allMetadata = topologyMetadata.hasNamedTopologies() ? rebuildMetadataForNamedTopologies(activePartitionHostMap, standbyPartitionHostMap) : rebuildMetadataForSingleTopology(activePartitionHostMap, standbyPartitionHostMap);
}