Examples with HostInfo org.apache.kafka.streams.state.HostInfo used on opensource projects

Example 36 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project apache-kafka-on-k8s by banzaicloud.

the class StreamsMetadataStateTest method shouldGetAllStreamsInstancesWithNoStores.

@Test
public void shouldGetAllStreamsInstancesWithNoStores() {
    builder.stream("topic-five").filter(new Predicate<Object, Object>() {

        @Override
        public boolean test(final Object key, final Object value) {
            return true;
        }
    }).to("some-other-topic");
    final TopicPartition tp5 = new TopicPartition("topic-five", 1);
    final HostInfo hostFour = new HostInfo("host-four", 8080);
    hostToPartitions.put(hostFour, Utils.mkSet(tp5));
    discovery.onChange(hostToPartitions, cluster.withPartitions(Collections.singletonMap(tp5, new PartitionInfo("topic-five", 1, null, null, null))));
    final StreamsMetadata expected = new StreamsMetadata(hostFour, Collections.singleton(globalTable), Collections.singleton(tp5));
    final Collection<StreamsMetadata> actual = discovery.getAllMetadata();
    assertTrue("expected " + actual + " to contain " + expected, actual.contains(expected));
}

Example 37 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method testOnAssignment.

@Test
public void testOnAssignment() throws Exception {
    configurePartitionAssignor(Collections.<String, Object>emptyMap());
    final List<TaskId> activeTaskList = Utils.mkList(task0, task3);
    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    final Map<TaskId, Set<TopicPartition>> standbyTasks = new HashMap<>();
    final Map<HostInfo, Set<TopicPartition>> hostState = Collections.singletonMap(new HostInfo("localhost", 9090), Utils.mkSet(t3p0, t3p3));
    activeTasks.put(task0, Utils.mkSet(t3p0));
    activeTasks.put(task3, Utils.mkSet(t3p3));
    standbyTasks.put(task1, Utils.mkSet(t3p1));
    standbyTasks.put(task2, Utils.mkSet(t3p2));
    final AssignmentInfo info = new AssignmentInfo(activeTaskList, standbyTasks, hostState);
    final PartitionAssignor.Assignment assignment = new PartitionAssignor.Assignment(Utils.mkList(t3p0, t3p3), info.encode());
    Capture<Cluster> capturedCluster = EasyMock.newCapture();
    taskManager.setPartitionsByHostState(hostState);
    EasyMock.expectLastCall();
    taskManager.setAssignmentMetadata(activeTasks, standbyTasks);
    EasyMock.expectLastCall();
    taskManager.setClusterMetadata(EasyMock.capture(capturedCluster));
    EasyMock.expectLastCall();
    EasyMock.replay(taskManager);
    partitionAssignor.onAssignment(assignment);
    EasyMock.verify(taskManager);
    assertEquals(Collections.singleton(t3p0.topic()), capturedCluster.getValue().topics());
    assertEquals(2, capturedCluster.getValue().partitionsForTopic(t3p0.topic()).size());
}

Example 38 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method shouldUpdateClusterMetadataAndHostInfoOnAssignment.

@Test
public void shouldUpdateClusterMetadataAndHostInfoOnAssignment() throws Exception {
    final TopicPartition partitionOne = new TopicPartition("topic", 1);
    final TopicPartition partitionTwo = new TopicPartition("topic", 2);
    final Map<HostInfo, Set<TopicPartition>> hostState = Collections.singletonMap(new HostInfo("localhost", 9090), Utils.mkSet(partitionOne, partitionTwo));
    configurePartitionAssignor(Collections.<String, Object>emptyMap());
    taskManager.setPartitionsByHostState(hostState);
    EasyMock.expectLastCall();
    EasyMock.replay(taskManager);
    partitionAssignor.onAssignment(createAssignment(hostState));
    EasyMock.verify(taskManager);
}

Example 39 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project apache-kafka-on-k8s by banzaicloud.

the class StreamsPartitionAssignorTest method shouldMapUserEndPointToTopicPartitions.

@Test
public void shouldMapUserEndPointToTopicPartitions() throws Exception {
    builder.setApplicationId(applicationId);
    builder.addSource(null, "source", null, null, null, "topic1");
    builder.addProcessor("processor", new MockProcessorSupplier(), "source");
    builder.addSink("sink", "output", null, null, null, "processor");
    final List<String> topics = Utils.mkList("topic1");
    final UUID uuid1 = UUID.randomUUID();
    mockTaskManager(Collections.<TaskId>emptySet(), Collections.<TaskId>emptySet(), uuid1, builder);
    configurePartitionAssignor(Collections.singletonMap(StreamsConfig.APPLICATION_SERVER_CONFIG, (Object) userEndPoint));
    partitionAssignor.setInternalTopicManager(new MockInternalTopicManager(streamsConfig, mockClientSupplier.restoreConsumer));
    final Map<String, PartitionAssignor.Subscription> subscriptions = new HashMap<>();
    final Set<TaskId> emptyTasks = Collections.emptySet();
    subscriptions.put("consumer1", new PartitionAssignor.Subscription(topics, new SubscriptionInfo(uuid1, emptyTasks, emptyTasks, userEndPoint).encode()));
    final Map<String, PartitionAssignor.Assignment> assignments = partitionAssignor.assign(metadata, subscriptions);
    final PartitionAssignor.Assignment consumerAssignment = assignments.get("consumer1");
    final AssignmentInfo assignmentInfo = AssignmentInfo.decode(consumerAssignment.userData());
    final Set<TopicPartition> topicPartitions = assignmentInfo.partitionsByHost().get(new HostInfo("localhost", 8080));
    assertEquals(Utils.mkSet(new TopicPartition("topic1", 0), new TopicPartition("topic1", 1), new TopicPartition("topic1", 2)), topicPartitions);
}

Example 40 with HostInfo

use of org.apache.kafka.streams.state.HostInfo in project kafka by apache.

the class StreamsPartitionAssignor method onAssignment.

@Override
public void onAssignment(final Assignment assignment, final ConsumerGroupMetadata metadata) {
    final List<TopicPartition> partitions = new ArrayList<>(assignment.partitions());
    partitions.sort(PARTITION_COMPARATOR);
    final AssignmentInfo info = AssignmentInfo.decode(assignment.userData());
    if (info.errCode() != AssignorError.NONE.code()) {
        // set flag to shutdown streams app
        assignmentErrorCode.set(info.errCode());
        return;
    }
    /*
         * latestCommonlySupportedVersion belongs to [usedSubscriptionMetadataVersion, LATEST_SUPPORTED_VERSION]
         * receivedAssignmentMetadataVersion belongs to [EARLIEST_PROBEABLE_VERSION, usedSubscriptionMetadataVersion]
         *
         * usedSubscriptionMetadataVersion will be downgraded to receivedAssignmentMetadataVersion during a rolling
         * bounce upgrade with version probing.
         *
         * usedSubscriptionMetadataVersion will be upgraded to latestCommonlySupportedVersion when all members have
         * been bounced and it is safe to use the latest version.
         */
    final int receivedAssignmentMetadataVersion = info.version();
    final int latestCommonlySupportedVersion = info.commonlySupportedVersion();
    validateMetadataVersions(receivedAssignmentMetadataVersion, latestCommonlySupportedVersion);
    // version 1 field
    final Map<TaskId, Set<TopicPartition>> activeTasks;
    // version 2 fields
    final Map<TopicPartition, PartitionInfo> topicToPartitionInfo;
    final Map<HostInfo, Set<TopicPartition>> partitionsByHost;
    final Map<HostInfo, Set<TopicPartition>> standbyPartitionsByHost;
    final long encodedNextScheduledRebalanceMs;
    switch(receivedAssignmentMetadataVersion) {
        case 1:
            validateActiveTaskEncoding(partitions, info, logPrefix);
            activeTasks = getActiveTasks(partitions, info);
            partitionsByHost = Collections.emptyMap();
            standbyPartitionsByHost = Collections.emptyMap();
            topicToPartitionInfo = Collections.emptyMap();
            encodedNextScheduledRebalanceMs = Long.MAX_VALUE;
            break;
        case 2:
        case 3:
        case 4:
        case 5:
            validateActiveTaskEncoding(partitions, info, logPrefix);
            activeTasks = getActiveTasks(partitions, info);
            partitionsByHost = info.partitionsByHost();
            standbyPartitionsByHost = Collections.emptyMap();
            topicToPartitionInfo = getTopicPartitionInfo(partitionsByHost);
            encodedNextScheduledRebalanceMs = Long.MAX_VALUE;
            break;
        case 6:
            validateActiveTaskEncoding(partitions, info, logPrefix);
            activeTasks = getActiveTasks(partitions, info);
            partitionsByHost = info.partitionsByHost();
            standbyPartitionsByHost = info.standbyPartitionByHost();
            topicToPartitionInfo = getTopicPartitionInfo(partitionsByHost);
            encodedNextScheduledRebalanceMs = Long.MAX_VALUE;
            break;
        case 7:
        case 8:
        case 9:
        case 10:
            validateActiveTaskEncoding(partitions, info, logPrefix);
            activeTasks = getActiveTasks(partitions, info);
            partitionsByHost = info.partitionsByHost();
            standbyPartitionsByHost = info.standbyPartitionByHost();
            topicToPartitionInfo = getTopicPartitionInfo(partitionsByHost);
            encodedNextScheduledRebalanceMs = info.nextRebalanceMs();
            break;
        default:
            throw new IllegalStateException("This code should never be reached." + " Please file a bug report at https://issues.apache.org/jira/projects/KAFKA/");
    }
    maybeScheduleFollowupRebalance(encodedNextScheduledRebalanceMs, receivedAssignmentMetadataVersion, latestCommonlySupportedVersion, partitionsByHost.keySet());
    final Cluster fakeCluster = Cluster.empty().withPartitions(topicToPartitionInfo);
    streamsMetadataState.onChange(partitionsByHost, standbyPartitionsByHost, fakeCluster);
    // we do not capture any exceptions but just let the exception thrown from consumer.poll directly
    // since when stream thread captures it, either we close all tasks as dirty or we close thread
    taskManager.handleAssignment(activeTasks, info.standbyTasks());
}