Examples with JoinGroupRequest org.apache.kafka.common.requests.JoinGroupRequest used on opensource projects

Example 6 with JoinGroupRequest

use of org.apache.kafka.common.requests.JoinGroupRequest in project kafka by apache.

the class AbstractCoordinatorTest method testNoGenerationWillNotTriggerProtocolNameCheck.

@Test
public void testNoGenerationWillNotTriggerProtocolNameCheck() {
    final String wrongProtocolName = "wrong-name";
    setupCoordinator();
    mockClient.reset();
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(mockTime.timer(0));
    mockClient.prepareResponse(body -> {
        if (!(body instanceof JoinGroupRequest)) {
            return false;
        }
        JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
        return joinGroupRequest.data().protocolType().equals(PROTOCOL_TYPE);
    }, joinGroupFollowerResponse(defaultGeneration, memberId, "memberid", Errors.NONE, PROTOCOL_TYPE));
    mockClient.prepareResponse(body -> {
        if (!(body instanceof SyncGroupRequest)) {
            return false;
        }
        coordinator.resetGenerationOnLeaveGroup();
        SyncGroupRequest syncGroupRequest = (SyncGroupRequest) body;
        return syncGroupRequest.data().protocolType().equals(PROTOCOL_TYPE) && syncGroupRequest.data().protocolName().equals(PROTOCOL_NAME);
    }, syncGroupResponse(Errors.NONE, PROTOCOL_TYPE, wrongProtocolName));
    // let the retry to complete successfully to break out of the while loop
    mockClient.prepareResponse(body -> {
        if (!(body instanceof JoinGroupRequest)) {
            return false;
        }
        JoinGroupRequest joinGroupRequest = (JoinGroupRequest) body;
        return joinGroupRequest.data().protocolType().equals(PROTOCOL_TYPE);
    }, joinGroupFollowerResponse(1, memberId, "memberid", Errors.NONE, PROTOCOL_TYPE));
    mockClient.prepareResponse(body -> {
        if (!(body instanceof SyncGroupRequest)) {
            return false;
        }
        SyncGroupRequest syncGroupRequest = (SyncGroupRequest) body;
        return syncGroupRequest.data().protocolType().equals(PROTOCOL_TYPE) && syncGroupRequest.data().protocolName().equals(PROTOCOL_NAME);
    }, syncGroupResponse(Errors.NONE, PROTOCOL_TYPE, PROTOCOL_NAME));
    // No exception shall be thrown as the generation is reset.
    coordinator.joinGroupIfNeeded(mockTime.timer(100L));
}

Example 7 with JoinGroupRequest

use of org.apache.kafka.common.requests.JoinGroupRequest in project kafka by apache.

the class AbstractCoordinatorTest method testWakeupAfterJoinGroupSent.

@Test
public void testWakeupAfterJoinGroupSent() throws Exception {
    setupCoordinator();
    mockClient.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    mockClient.prepareResponse(new MockClient.RequestMatcher() {

        private int invocations = 0;

        @Override
        public boolean matches(AbstractRequest body) {
            invocations++;
            boolean isJoinGroupRequest = body instanceof JoinGroupRequest;
            if (isJoinGroupRequest && invocations == 1)
                // simulate wakeup before the request returns
                throw new WakeupException();
            return isJoinGroupRequest;
        }
    }, joinGroupFollowerResponse(1, memberId, leaderId, Errors.NONE));
    mockClient.prepareResponse(syncGroupResponse(Errors.NONE));
    AtomicBoolean heartbeatReceived = prepareFirstHeartbeat();
    try {
        coordinator.ensureActiveGroup();
        fail("Should have woken up from ensureActiveGroup()");
    } catch (WakeupException ignored) {
    }
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(0, coordinator.onJoinCompleteInvokes);
    assertFalse(heartbeatReceived.get());
    coordinator.ensureActiveGroup();
    assertEquals(1, coordinator.onJoinPrepareInvokes);
    assertEquals(1, coordinator.onJoinCompleteInvokes);
    awaitFirstHeartbeat(heartbeatReceived);
}

Example 8 with JoinGroupRequest

use of org.apache.kafka.common.requests.JoinGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testMetadataRefreshDuringRebalance.

@Test
public void testMetadataRefreshDuringRebalance() {
    final String consumerId = "leader";
    final List<TopicPartition> owned = Collections.emptyList();
    final List<TopicPartition> oldAssigned = singletonList(t1p);
    subscriptions.subscribe(Pattern.compile(".*"), rebalanceListener);
    client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
    coordinator.maybeUpdateSubscriptionMetadata();
    assertEquals(singleton(topic1), subscriptions.subscription());
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    Map<String, List<String>> initialSubscription = singletonMap(consumerId, singletonList(topic1));
    partitionAssignor.prepare(singletonMap(consumerId, oldAssigned));
    // the metadata will be updated in flight with a new topic added
    final List<String> updatedSubscription = Arrays.asList(topic1, topic2);
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE));
    client.prepareResponse(body -> {
        final Map<String, Integer> updatedPartitions = new HashMap<>();
        for (String topic : updatedSubscription) updatedPartitions.put(topic, 1);
        client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, updatedPartitions));
        return true;
    }, syncGroupResponse(oldAssigned, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    // rejoin will only be set in the next poll call
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(singleton(topic1), subscriptions.subscription());
    assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
    // nothing to be revoked and hence no callback triggered
    assertEquals(0, rebalanceListener.revokedCount);
    assertNull(rebalanceListener.revoked);
    assertEquals(1, rebalanceListener.assignedCount);
    assertEquals(getAdded(owned, oldAssigned), rebalanceListener.assigned);
    List<TopicPartition> newAssigned = Arrays.asList(t1p, t2p);
    final Map<String, List<String>> updatedSubscriptions = singletonMap(consumerId, Arrays.asList(topic1, topic2));
    partitionAssignor.prepare(singletonMap(consumerId, newAssigned));
    // we expect to see a second rebalance with the new-found topics
    client.prepareResponse(body -> {
        JoinGroupRequest join = (JoinGroupRequest) body;
        Iterator<JoinGroupRequestData.JoinGroupRequestProtocol> protocolIterator = join.data().protocols().iterator();
        assertTrue(protocolIterator.hasNext());
        JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
        ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
        ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
        metadata.rewind();
        return subscription.topics().containsAll(updatedSubscription);
    }, joinGroupLeaderResponse(2, consumerId, updatedSubscriptions, Errors.NONE));
    // update the metadata again back to topic1
    client.prepareResponse(body -> {
        client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
        return true;
    }, syncGroupResponse(newAssigned, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    Collection<TopicPartition> revoked = getRevoked(oldAssigned, newAssigned);
    int revokedCount = revoked.isEmpty() ? 0 : 1;
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(toSet(updatedSubscription), subscriptions.subscription());
    assertEquals(toSet(newAssigned), subscriptions.assignedPartitions());
    assertEquals(revokedCount, rebalanceListener.revokedCount);
    assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
    assertEquals(2, rebalanceListener.assignedCount);
    assertEquals(getAdded(oldAssigned, newAssigned), rebalanceListener.assigned);
    // we expect to see a third rebalance with the new-found topics
    partitionAssignor.prepare(singletonMap(consumerId, oldAssigned));
    client.prepareResponse(body -> {
        JoinGroupRequest join = (JoinGroupRequest) body;
        Iterator<JoinGroupRequestData.JoinGroupRequestProtocol> protocolIterator = join.data().protocols().iterator();
        assertTrue(protocolIterator.hasNext());
        JoinGroupRequestData.JoinGroupRequestProtocol protocolMetadata = protocolIterator.next();
        ByteBuffer metadata = ByteBuffer.wrap(protocolMetadata.metadata());
        ConsumerPartitionAssignor.Subscription subscription = ConsumerProtocol.deserializeSubscription(metadata);
        metadata.rewind();
        return subscription.topics().contains(topic1);
    }, joinGroupLeaderResponse(3, consumerId, initialSubscription, Errors.NONE));
    client.prepareResponse(syncGroupResponse(oldAssigned, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    revoked = getRevoked(newAssigned, oldAssigned);
    assertFalse(revoked.isEmpty());
    revokedCount += 1;
    Collection<TopicPartition> added = getAdded(newAssigned, oldAssigned);
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(singleton(topic1), subscriptions.subscription());
    assertEquals(toSet(oldAssigned), subscriptions.assignedPartitions());
    assertEquals(revokedCount, rebalanceListener.revokedCount);
    assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
    assertEquals(3, rebalanceListener.assignedCount);
    assertEquals(added, rebalanceListener.assigned);
    assertEquals(0, rebalanceListener.lostCount);
}

Example 9 with JoinGroupRequest

use of org.apache.kafka.common.requests.JoinGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testRebalanceWithMetadataChange.

/**
 * Verifies that the consumer re-joins after a metadata change. If JoinGroup fails
 * and metadata reverts to its original value, the consumer should still retry JoinGroup.
 */
@Test
public void testRebalanceWithMetadataChange() {
    final String consumerId = "leader";
    final List<String> topics = Arrays.asList(topic1, topic2);
    final List<TopicPartition> partitions = Arrays.asList(t1p, t2p);
    subscriptions.subscribe(toSet(topics), rebalanceListener);
    client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, Utils.mkMap(Utils.mkEntry(topic1, 1), Utils.mkEntry(topic2, 1))));
    coordinator.maybeUpdateSubscriptionMetadata();
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    Map<String, List<String>> initialSubscription = singletonMap(consumerId, topics);
    partitionAssignor.prepare(singletonMap(consumerId, partitions));
    client.prepareResponse(joinGroupLeaderResponse(1, consumerId, initialSubscription, Errors.NONE));
    client.prepareResponse(syncGroupResponse(partitions, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    // rejoin will only be set in the next poll call
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(toSet(topics), subscriptions.subscription());
    assertEquals(toSet(partitions), subscriptions.assignedPartitions());
    assertEquals(0, rebalanceListener.revokedCount);
    assertNull(rebalanceListener.revoked);
    assertEquals(1, rebalanceListener.assignedCount);
    // Change metadata to trigger rebalance.
    client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, singletonMap(topic1, 1)));
    coordinator.poll(time.timer(0));
    // Revert metadata to original value. Fail pending JoinGroup. Another
    // JoinGroup should be sent, which will be completed successfully.
    client.updateMetadata(RequestTestUtils.metadataUpdateWith(1, Utils.mkMap(Utils.mkEntry(topic1, 1), Utils.mkEntry(topic2, 1))));
    client.respond(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NOT_COORDINATOR));
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.poll(time.timer(0));
    assertTrue(coordinator.rejoinNeededOrPending());
    client.respond(request -> {
        if (!(request instanceof JoinGroupRequest)) {
            return false;
        } else {
            JoinGroupRequest joinRequest = (JoinGroupRequest) request;
            return consumerId.equals(joinRequest.data().memberId());
        }
    }, joinGroupLeaderResponse(2, consumerId, initialSubscription, Errors.NONE));
    client.prepareResponse(syncGroupResponse(partitions, Errors.NONE));
    coordinator.poll(time.timer(Long.MAX_VALUE));
    assertFalse(coordinator.rejoinNeededOrPending());
    Collection<TopicPartition> revoked = getRevoked(partitions, partitions);
    assertEquals(revoked.isEmpty() ? 0 : 1, rebalanceListener.revokedCount);
    assertEquals(revoked.isEmpty() ? null : revoked, rebalanceListener.revoked);
    // No partitions have been lost since the rebalance failure was not fatal
    assertEquals(0, rebalanceListener.lostCount);
    assertNull(rebalanceListener.lost);
    Collection<TopicPartition> added = getAdded(partitions, partitions);
    assertEquals(2, rebalanceListener.assignedCount);
    assertEquals(added.isEmpty() ? Collections.emptySet() : toSet(partitions), rebalanceListener.assigned);
    assertEquals(toSet(partitions), subscriptions.assignedPartitions());
}

Example 10 with JoinGroupRequest

use of org.apache.kafka.common.requests.JoinGroupRequest in project kafka by apache.

the class ConsumerCoordinatorTest method testUnknownMemberIdOnSyncGroup.

@Test
public void testUnknownMemberIdOnSyncGroup() {
    subscriptions.subscribe(singleton(topic1), rebalanceListener);
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // join initially, but let coordinator returns unknown member id
    client.prepareResponse(joinGroupFollowerResponse(1, consumerId, "leader", Errors.NONE));
    client.prepareResponse(syncGroupResponse(Collections.emptyList(), Errors.UNKNOWN_MEMBER_ID));
    // now we should see a new join with the empty UNKNOWN_MEMBER_ID
    client.prepareResponse(body -> {
        JoinGroupRequest joinRequest = (JoinGroupRequest) body;
        return joinRequest.data().memberId().equals(JoinGroupRequest.UNKNOWN_MEMBER_ID);
    }, joinGroupFollowerResponse(2, consumerId, "leader", Errors.NONE));
    client.prepareResponse(syncGroupResponse(singletonList(t1p), Errors.NONE));
    coordinator.joinGroupIfNeeded(time.timer(Long.MAX_VALUE));
    assertFalse(coordinator.rejoinNeededOrPending());
    assertEquals(singleton(t1p), subscriptions.assignedPartitions());
}