Examples with SyncGroupRequest org.apache.kafka.common.requests.SyncGroupRequest used on opensource projects

Example 16 with SyncGroupRequest

use of org.apache.kafka.common.requests.SyncGroupRequest in project apache-kafka-on-k8s by banzaicloud.

the class ConsumerCoordinatorTest method testUpdateMetadataDuringRebalance.

@Test
public void testUpdateMetadataDuringRebalance() {
    final String topic1 = "topic1";
    final String topic2 = "topic2";
    TopicPartition tp1 = new TopicPartition(topic1, 0);
    TopicPartition tp2 = new TopicPartition(topic2, 0);
    final String consumerId = "leader";
    List<String> topics = Arrays.asList(topic1, topic2);
    subscriptions.subscribe(new HashSet<>(topics), rebalanceListener);
    metadata.setTopics(topics);
    // we only have metadata for one topic initially
    metadata.update(TestUtils.singletonCluster(topic1, 1), Collections.<String>emptySet(), time.milliseconds());
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady();
    // prepare initial rebalance
    Map<String, List<String>> memberSubscriptions = singletonMap(consumerId, topics);
    partitionAssignor.prepare(singletonMap(consumerId, Collections.singletonList(tp1)));
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
    client.prepareResponse(new MockClient.RequestMatcher() {

        @Override
        public boolean matches(AbstractRequest body) {
            SyncGroupRequest sync = (SyncGroupRequest) body;
            if (sync.memberId().equals(consumerId) && sync.generationId() == 1 && sync.groupAssignment().containsKey(consumerId)) {
                // trigger the metadata update including both topics after the sync group request has been sent
                Map<String, Integer> topicPartitionCounts = new HashMap<>();
                topicPartitionCounts.put(topic1, 1);
                topicPartitionCounts.put(topic2, 1);
                metadata.update(TestUtils.singletonCluster(topicPartitionCounts), Collections.<String>emptySet(), time.milliseconds());
                return true;
            }
            return false;
        }
    }, syncGroupResponse(Collections.singletonList(tp1), Errors.NONE));
    // the metadata update should trigger a second rebalance
    client.prepareResponse(joinGroupLeaderResponse(2, consumerId, memberSubscriptions, Errors.NONE));
    client.prepareResponse(syncGroupResponse(Arrays.asList(tp1, tp2), Errors.NONE));
    coordinator.poll(time.milliseconds(), Long.MAX_VALUE);
    assertFalse(coordinator.needRejoin());
    assertEquals(new HashSet<>(Arrays.asList(tp1, tp2)), subscriptions.assignedPartitions());
}

Example 17 with SyncGroupRequest

use of org.apache.kafka.common.requests.SyncGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testOutdatedCoordinatorAssignment.

@Test
public void testOutdatedCoordinatorAssignment() {
    final String consumerId = "outdated_assignment";
    final List<TopicPartition> owned = Collections.emptyList();
    final List<String> oldSubscription = singletonList(topic2);
    final List<TopicPartition> oldAssignment = Arrays.asList(t2p);
    final List<String> newSubscription = singletonList(topic1);
    final List<TopicPartition> newAssignment = Arrays.asList(t1p);
    subscriptions.subscribe(toSet(oldSubscription), rebalanceListener);
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // Test coordinator returning unsubscribed partitions
    partitionAssignor.prepare(singletonMap(consumerId, newAssignment));
    // First incorrect assignment for subscription
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, singletonMap(consumerId, oldSubscription), Errors.NONE));
    client.prepareResponse(body -> {
        SyncGroupRequest sync = (SyncGroupRequest) body;
        return sync.data().memberId().equals(consumerId) && sync.data().generationId() == 1 && sync.groupAssignments().containsKey(consumerId);
    }, syncGroupResponse(oldAssignment, Errors.NONE));
    // Second correct assignment for subscription
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, singletonMap(consumerId, newSubscription), Errors.NONE));
    client.prepareResponse(body -> {
        SyncGroupRequest sync = (SyncGroupRequest) body;
        return sync.data().memberId().equals(consumerId) && sync.data().generationId() == 1 && sync.groupAssignments().containsKey(consumerId);
    }, syncGroupResponse(newAssignment, Errors.NONE));
    // Poll once so that the join group future gets created and complete
    coordinator.poll(time.timer(0));
    // Before the sync group response gets completed change the subscription
    subscriptions.subscribe(toSet(newSubscription), rebalanceListener);
    coordinator.poll(time.timer(0));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    final Collection<TopicPartition> assigned = getAdded(owned, newAssignment);
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(toSet(newAssignment), subscriptions.assignedPartitions());
    assertEquals(toSet(newSubscription), subscriptions.metadataTopics());
    assertEquals(protocol == EAGER ? 1 : 0, rebalanceListener.revokedCount);
    assertEquals(1, rebalanceListener.assignedCount);
    assertEquals(assigned, rebalanceListener.assigned);
}

Example 18 with SyncGroupRequest

use of org.apache.kafka.common.requests.SyncGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testForceMetadataRefreshForPatternSubscriptionDuringRebalance.

@Test
public void testForceMetadataRefreshForPatternSubscriptionDuringRebalance() {
    // Set up a non-leader consumer with pattern subscription and a cluster containing one topic matching the
    // pattern.
    subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener);
    client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
    coordinator.maybeUpdateSubscriptionMetadata();
    assertEquals(singleton(topic1), subscriptions.subscription());
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // Instrument the test so that metadata will contain two topics after next refresh.
    client.prepareMetadataUpdate(metadataResponse);
    client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
    client.prepareResponse(body -> {
        SyncGroupRequest sync = (SyncGroupRequest) body;
        return sync.data().memberId().equals(consumerId) && sync.data().generationId() == 1 && sync.groupAssignments().isEmpty();
    }, syncGroupResponse(singletonList(t1p), Errors.NONE));
    partitionAssignor.prepare(singletonMap(consumerId, singletonList(t1p)));
    // This will trigger rebalance.
    coordinator.poll(time.timer(Long.MAX_VALUE));
    // Make sure that the metadata was refreshed during the rebalance and thus subscriptions now contain two topics.
    final Set<String> updatedSubscriptionSet = new HashSet<>(Arrays.asList(topic1, topic2));
    assertEquals(updatedSubscriptionSet, subscriptions.subscription());
    // Refresh the metadata again. Since there have been no changes since the last refresh, it won't trigger
    // rebalance again.
    metadata.requestUpdate();
    consumerClient.poll(time.timer(Long.MAX_VALUE));
    assertFalse(coordinator.rejoinNeededOrPending());
}

Example 19 with SyncGroupRequest

use of org.apache.kafka.common.requests.SyncGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testNormalJoinGroupLeader.

@Test
public void testNormalJoinGroupLeader() {
    final String consumerId = "leader";
    final Set<String> subscription = singleton(topic1);
    final List<TopicPartition> owned = Collections.emptyList();
    final List<TopicPartition> assigned = Arrays.asList(t1p);
    subscriptions.subscribe(singleton(topic1), rebalanceListener);
    // ensure metadata is up-to-date for leader
    client.updateMetadata(metadataResponse);
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // normal join group
    Map<String, List<String>> memberSubscriptions = singletonMap(consumerId, singletonList(topic1));
    partitionAssignor.prepare(singletonMap(consumerId, assigned));
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, memberSubscriptions, Errors.NONE));
    client.prepareResponse(body -> {
        SyncGroupRequest sync = (SyncGroupRequest) body;
        return sync.data().memberId().equals(consumerId) && sync.data().generationId() == 1 && sync.groupAssignments().containsKey(consumerId);
    }, syncGroupResponse(assigned, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(toSet(assigned), subscriptions.assignedPartitions());
    assertEquals(subscription, subscriptions.metadataTopics());
    assertEquals(0, rebalanceListener.revokedCount);
    assertNull(rebalanceListener.revoked);
    assertEquals(1, rebalanceListener.assignedCount);
    assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
}

Example 20 with SyncGroupRequest

use of org.apache.kafka.common.requests.SyncGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testNormalJoinGroupFollower.

@Test
public void testNormalJoinGroupFollower() {
    final Set<String> subscription = singleton(topic1);
    final List<TopicPartition> owned = Collections.emptyList();
    final List<TopicPartition> assigned = singletonList(t1p);
    subscriptions.subscribe(subscription, rebalanceListener);
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // normal join group
    client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
    client.prepareResponse(body -> {
        SyncGroupRequest sync = (SyncGroupRequest) body;
        return sync.data().memberId().equals(consumerId) && sync.data().generationId() == 1 && sync.groupAssignments().isEmpty();
    }, syncGroupResponse(assigned, Errors.NONE));
    coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(toSet(assigned), subscriptions.assignedPartitions());
    assertEquals(subscription, subscriptions.metadataTopics());
    assertEquals(0, rebalanceListener.revokedCount);
    assertNull(rebalanceListener.revoked);
    assertEquals(1, rebalanceListener.assignedCount);
    assertEquals(getAdded(owned, assigned), rebalanceListener.assigned);
}