Examples with HostInfo org.apache.kafka.streams.state.HostInfo used on opensource projects

Example 41 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsMetadataState method getKeyQueryMetadataForKey.

private <K> KeyQueryMetadata getKeyQueryMetadataForKey(final String storeName, final K key, final StreamPartitioner<? super K, ?> partitioner, final SourceTopicsInfo sourceTopicsInfo) {
    final Integer partition = partitioner.partition(sourceTopicsInfo.topicWithMostPartitions, key, null, sourceTopicsInfo.maxPartitions);
    final Set<TopicPartition> matchingPartitions = new HashSet<>();
    for (final String sourceTopic : sourceTopicsInfo.sourceTopics) {
        matchingPartitions.add(new TopicPartition(sourceTopic, partition));
    }
    HostInfo activeHost = UNKNOWN_HOST;
    final Set<HostInfo> standbyHosts = new HashSet<>();
    for (final StreamsMetadata streamsMetadata : allMetadata) {
        final Set<String> activeStateStoreNames = streamsMetadata.stateStoreNames();
        final Set<TopicPartition> topicPartitions = new HashSet<>(streamsMetadata.topicPartitions());
        final Set<String> standbyStateStoreNames = streamsMetadata.standbyStateStoreNames();
        final Set<TopicPartition> standbyTopicPartitions = new HashSet<>(streamsMetadata.standbyTopicPartitions());
        topicPartitions.retainAll(matchingPartitions);
        if (activeStateStoreNames.contains(storeName) && !topicPartitions.isEmpty()) {
            activeHost = streamsMetadata.hostInfo();
        }
        standbyTopicPartitions.retainAll(matchingPartitions);
        if (standbyStateStoreNames.contains(storeName) && !standbyTopicPartitions.isEmpty()) {
            standbyHosts.add(streamsMetadata.hostInfo());
        }
    }
    return new KeyQueryMetadata(activeHost, standbyHosts, partition);
}

Example 42 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class AssignmentInfo method decodeHostPartitionMapUsingDictionary.

private static Map<HostInfo, Set<TopicPartition>> decodeHostPartitionMapUsingDictionary(final DataInputStream in, final Map<Integer, String> topicIndexDict) throws IOException {
    final Map<HostInfo, Set<TopicPartition>> hostPartitionMap = new HashMap<>();
    final int numEntries = in.readInt();
    for (int i = 0; i < numEntries; i++) {
        final HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
        hostPartitionMap.put(hostInfo, readTopicPartitions(in, topicIndexDict));
    }
    return hostPartitionMap;
}

Example 43 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class AssignmentInfo method decodePartitionsByHost.

private static void decodePartitionsByHost(final AssignmentInfo assignmentInfo, final DataInputStream in) throws IOException {
    assignmentInfo.partitionsByHost = new HashMap<>();
    final int numEntries = in.readInt();
    for (int i = 0; i < numEntries; i++) {
        final HostInfo hostInfo = new HostInfo(in.readUTF(), in.readInt());
        assignmentInfo.partitionsByHost.put(hostInfo, readTopicPartitions(in));
    }
}

Example 44 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.

@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() {
    final Map<String, Object> props = configProps();
    props.put(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, 1);
    props.put(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT);
    final StreamsBuilder streamsBuilder = new StreamsBuilder();
    streamsBuilder.stream("topic1").groupByKey().count();
    builder = TopologyWrapper.getInternalTopologyBuilder(streamsBuilder.build());
    topologyMetadata = new TopologyMetadata(builder, new StreamsConfig(props));
    createDefaultMockTaskManager();
    adminClient = createMockAdminClientForAssignor(getTopicPartitionOffsetsMap(singletonList(APPLICATION_ID + "-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog"), singletonList(3)));
    configurePartitionAssignorWith(props);
    subscriptions.put("consumer1", new Subscription(Collections.singletonList("topic1"), getInfo(UUID_1, EMPTY_TASKS, EMPTY_TASKS, USER_END_POINT).encode()));
    subscriptions.put("consumer2", new Subscription(Collections.singletonList("topic1"), getInfo(UUID_2, EMPTY_TASKS, EMPTY_TASKS, OTHER_END_POINT).encode()));
    final Set<TopicPartition> allPartitions = mkSet(t1p0, t1p1, t1p2);
    final Map<String, Assignment> assign = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final Assignment consumer1Assignment = assign.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
    final Set<TopicPartition> consumer1ActivePartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    final Set<TopicPartition> consumer2ActivePartitions = assignmentInfo.partitionsByHost().get(new HostInfo("other", 9090));
    final Set<TopicPartition> consumer1StandbyPartitions = assignmentInfo.standbyPartitionByHost().get(new HostInfo("localhost", 8080));
    final Set<TopicPartition> consumer2StandbyPartitions = assignmentInfo.standbyPartitionByHost().get(new HostInfo("other", 9090));
    final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1ActivePartitions);
    allAssignedPartitions.addAll(consumer2ActivePartitions);
    assertThat(consumer1ActivePartitions, not(allPartitions));
    assertThat(consumer2ActivePartitions, not(allPartitions));
    assertThat(consumer1ActivePartitions, equalTo(consumer2StandbyPartitions));
    assertThat(consumer2ActivePartitions, equalTo(consumer1StandbyPartitions));
    assertThat(allAssignedPartitions, equalTo(allPartitions));
}

Example 45 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsPartitionAssignorTest method shouldMapUserEndPointToTopicPartitions.

@Test
public void shouldMapUserEndPointToTopicPartitions() {
    builder.addSource(null, "source", null, null, null, "topic1");
    builder.addProcessor("processor", new MockApiProcessorSupplier<>(), "source");
    builder.addSink("sink", "output", null, null, null, "processor");
    final List<String> topics = Collections.singletonList("topic1");
    createDefaultMockTaskManager();
    configurePartitionAssignorWith(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, USER_END_POINT));
    subscriptions.put("consumer1", new Subscription(topics, getInfo(UUID_1, EMPTY_TASKS, EMPTY_TASKS, USER_END_POINT).encode()));
    final Map<String, Assignment> assignments = partitionAssignor.assign(metadata, new GroupSubscription(subscriptions)).groupAssignment();
    final Assignment consumerAssignment = assignments.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumerAssignment.userData());
    final Set<TopicPartition> topicPartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    assertEquals(mkSet(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2)), topicPartitions);
}