Examples with HostInfo org.apache.kafka.streams.state.HostInfo used on opensource projects

Example 1 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamPartitionAssignorTest method testOnAssignment.

@Test
public void testOnAssignment() throws Exception {
    TopicPartition t2p3 = new TopicPartition("topic2", 3);
    TopologyBuilder builder = new TopologyBuilder();
    builder.addSource("source1", "topic1");
    builder.addSource("source2", "topic2");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source1", "source2");
    UUID uuid = UUID.randomUUID();
    String client1 = "client1";
    StreamThread thread = new StreamThread(builder, config, mockClientSupplier, "test", client1, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(thread, "test", client1));
    List<TaskId> activeTaskList = Utils.mkList(task0, task3);
    Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    activeTasks.put(task0, Utils.mkSet(t1p0));
    activeTasks.put(task3, Utils.mkSet(t2p3));
    standbyTasks.put(task1, Utils.mkSet(t1p0));
    standbyTasks.put(task2, Utils.mkSet(t2p0));
    AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, new HashMap<HostInfo, Set<TopicPartition>>());
    PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(Utils.mkList(t1p0, t2p3), info.encode());
    partitionAssignor.onAssignment(assignment);
    assertEquals(activeTasks, partitionAssignor.activeTasks());
    assertEquals(standbyTasks, partitionAssignor.standbyTasks());
}

Example 2 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamPartitionAssignorTest method shouldUpdatePartitionHostInfoMapOnAssignment.

@Test
public void shouldUpdatePartitionHostInfoMapOnAssignment() throws Exception {
    final TopicPartition partitionOne = new TopicPartition("topic", 1);
    final TopicPartition partitionTwo = new TopicPartition("topic", 2);
    final Map<HostInfo, Set<TopicPartition>> firstHostState = Collections.singletonMap(new HostInfo("localhost", 9090), Utils.mkSet(partitionOne, partitionTwo));
    final Map<HostInfo, Set<TopicPartition>> secondHostState = new HashMap<>();
    secondHostState.put(new HostInfo("localhost", 9090), Utils.mkSet(partitionOne));
    secondHostState.put(new HostInfo("other", 9090), Utils.mkSet(partitionTwo));
    partitionAssignor.onAssignment(createAssignment(firstHostState));
    assertEquals(firstHostState, partitionAssignor.getPartitionsByHostState());
    partitionAssignor.onAssignment(createAssignment(secondHostState));
    assertEquals(secondHostState, partitionAssignor.getPartitionsByHostState());
}

Example 3 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamPartitionAssignorTest method shouldUpdateClusterMetadataOnAssignment.

@Test
public void shouldUpdateClusterMetadataOnAssignment() throws Exception {
    final TopicPartition topicOne = new TopicPartition("topic", 1);
    final TopicPartition topicTwo = new TopicPartition("topic2", 2);
    final Map<HostInfo, Set<TopicPartition>> firstHostState = Collections.singletonMap(new HostInfo("localhost", 9090), Utils.mkSet(topicOne));
    final Map<HostInfo, Set<TopicPartition>> secondHostState = Collections.singletonMap(new HostInfo("localhost", 9090), Utils.mkSet(topicOne, topicTwo));
    partitionAssignor.onAssignment(createAssignment(firstHostState));
    assertEquals(Utils.mkSet("topic"), partitionAssignor.clusterMetadata().topics());
    partitionAssignor.onAssignment(createAssignment(secondHostState));
    assertEquals(Utils.mkSet("topic", "topic2"), partitionAssignor.clusterMetadata().topics());
}

Example 4 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamPartitionAssignorTest method shouldExposeHostStateToTopicPartitionsOnAssignment.

@Test
public void shouldExposeHostStateToTopicPartitionsOnAssignment() throws Exception {
    List<TopicPartition> topic = Collections.singletonList(new TopicPartition("topic", 0));
    final Map<HostInfo, Set<TopicPartition>> hostState = Collections.singletonMap(new HostInfo("localhost", 80), Collections.singleton(new TopicPartition("topic", 0)));
    AssignmentInfo assignmentInfo = new AssignmentInfo(Collections.singletonList(new TaskId(0, 0)), Collections.<TaskId, Set<TopicPartition>>emptyMap(), hostState);
    partitionAssignor.onAssignment(new PartitionAssignor.Assignment(topic, assignmentInfo.encode()));
    assertEquals(hostState, partitionAssignor.getPartitionsByHostState());
}

Example 5 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamPartitionAssignorTest method shouldNotAddStandbyTaskPartitionsToPartitionsForHost.

@Test
public void shouldNotAddStandbyTaskPartitionsToPartitionsForHost() throws Exception {
    final Properties props = configProps();
    props.setProperty(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, "1");
    final StreamsConfig config = new StreamsConfig(props);
    final KStreamBuilder builder = new KStreamBuilder();
    final String applicationId = "appId";
    builder.setApplicationId(applicationId);
    builder.stream("topic1").groupByKey().count("count");
    final UUID uuid = UUID.randomUUID();
    final String client = "client1";
    final StreamThread streamThread = new StreamThread(builder, config, mockClientSupplier, applicationId, client, uuid, new Metrics(), Time.SYSTEM, new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0);
    partitionAssignor.configure(config.getConsumerConfigs(streamThread, applicationId, client));
    partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamThread.config, mockClientSupplier.restoreConsumer));
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put("consumer1", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(uuid, emptyTasks, emptyTasks, userEndPoint).encode()));
    subscriptions.put("consumer2", new PartitionAssignor.Subscription(Collections.singletonList("topic1"), new SubscriptionInfo(UUID.randomUUID(), emptyTasks, emptyTasks, "other:9090").encode()));
    final Set<TopicPartition> allPartitions = Utils.mkSet(t1p0, t1p1, t1p2);
    final Map<String, PartitionAssignor.Assignment> assign = partitionAssignor.assign(metadata, subscriptions);
    final PartitionAssignor.Assignment consumer1Assignment = assign.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumer1Assignment.userData());
    final Set<TopicPartition> consumer1partitions = assignmentInfo.partitionsByHost.get(new HostInfo("localhost", 2171));
    final Set<TopicPartition> consumer2Partitions = assignmentInfo.partitionsByHost.get(new HostInfo("other", 9090));
    final HashSet<TopicPartition> allAssignedPartitions = new HashSet<>(consumer1partitions);
    allAssignedPartitions.addAll(consumer2Partitions);
    assertThat(consumer1partitions, not(allPartitions));
    assertThat(consumer2Partitions, not(allPartitions));
    assertThat(allAssignedPartitions, equalTo(allPartitions));
}