Examples with Assignment org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment used on opensource projects

Example 16 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class ConsumerCoordinator method maybeUpdateGroupSubscription.

/**
 * user-customized assignor may have created some topics that are not in the subscription list
 * and assign their partitions to the members; in this case we would like to update the leader's
 * own metadata with the newly added topics so that it will not trigger a subsequent rebalance
 * when these topics gets updated from metadata refresh.
 *
 * We skip the check for in-product assignors since this will not happen in in-product assignors.
 *
 * TODO: this is a hack and not something we want to support long-term unless we push regex into the protocol
 *       we may need to modify the ConsumerPartitionAssignor API to better support this case.
 *
 * @param assignorName          the selected assignor name
 * @param assignments           the assignments after assignor assigned
 * @param allSubscribedTopics   all consumers' subscribed topics
 */
private void maybeUpdateGroupSubscription(String assignorName, Map<String, Assignment> assignments, Set<String> allSubscribedTopics) {
    if (!isAssignFromSubscribedTopicsAssignor(assignorName)) {
        Set<String> assignedTopics = new HashSet<>();
        for (Assignment assigned : assignments.values()) {
            for (TopicPartition tp : assigned.partitions()) assignedTopics.add(tp.topic());
        }
        if (!assignedTopics.containsAll(allSubscribedTopics)) {
            SortedSet<String> notAssignedTopics = new TreeSet<>(allSubscribedTopics);
            notAssignedTopics.removeAll(assignedTopics);
            log.warn("The following subscribed topics are not assigned to any members: {} ", notAssignedTopics);
        }
        if (!allSubscribedTopics.containsAll(assignedTopics)) {
            SortedSet<String> newlyAddedTopics = new TreeSet<>(assignedTopics);
            newlyAddedTopics.removeAll(allSubscribedTopics);
            log.info("The following not-subscribed topics are assigned, and their metadata will be " + "fetched from the brokers: {}", newlyAddedTopics);
            allSubscribedTopics.addAll(newlyAddedTopics);
            updateGroupSubscription(allSubscribedTopics);
        }
    }
}

Example 17 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class ConsumerProtocolTest method deserializeNullAssignmentUserData.

@Test
public void deserializeNullAssignmentUserData() {
    List<TopicPartition> partitions = Arrays.asList(tp1, tp2);
    ByteBuffer buffer = ConsumerProtocol.serializeAssignment(new Assignment(partitions, null));
    Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
    assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
    assertNull(parsedAssignment.userData());
}

Example 18 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class ConsumerProtocolTest method serializeDeserializeAssignmentAllVersions.

@Test
public void serializeDeserializeAssignmentAllVersions() {
    List<TopicPartition> partitions = Arrays.asList(tp1, tp2);
    Assignment assignment = new Assignment(partitions, ByteBuffer.wrap("hello".getBytes()));
    for (short version = ConsumerProtocolAssignment.LOWEST_SUPPORTED_VERSION; version <= ConsumerProtocolAssignment.HIGHEST_SUPPORTED_VERSION; version++) {
        ByteBuffer buffer = ConsumerProtocol.serializeAssignment(assignment, version);
        Assignment parsedAssignment = ConsumerProtocol.deserializeAssignment(buffer);
        assertEquals(toSet(partitions), toSet(parsedAssignment.partitions()));
        assertEquals(assignment.userData(), parsedAssignment.userData());
    }
}

Example 19 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class ConsumerProtocolTest method deserializeFutureAssignmentVersion.

@Test
public void deserializeFutureAssignmentVersion() {
    // verify that a new version which adds a field is still parseable
    short version = 100;
    Schema assignmentSchemaV100 = new Schema(new Field("assigned_partitions", new ArrayOf(ConsumerProtocolAssignment.TopicPartition.SCHEMA_0)), new Field("user_data", Type.BYTES), new Field("foo", Type.STRING));
    Struct assignmentV100 = new Struct(assignmentSchemaV100);
    assignmentV100.set("assigned_partitions", new Object[] { new Struct(ConsumerProtocolAssignment.TopicPartition.SCHEMA_0).set("topic", tp1.topic()).set("partitions", new Object[] { tp1.partition() }) });
    assignmentV100.set("user_data", ByteBuffer.wrap(new byte[0]));
    assignmentV100.set("foo", "bar");
    Struct headerV100 = new Struct(new Schema(new Field("version", Type.INT16)));
    headerV100.set("version", version);
    ByteBuffer buffer = ByteBuffer.allocate(assignmentV100.sizeOf() + headerV100.sizeOf());
    headerV100.writeTo(buffer);
    assignmentV100.writeTo(buffer);
    buffer.flip();
    Assignment assignment = ConsumerProtocol.deserializeAssignment(buffer);
    assertEquals(toSet(Collections.singletonList(tp1)), toSet(assignment.partitions()));
}

Example 20 with Assignment

use of org.apache.kafka.clients.consumer.ConsumerPartitionAssignor.Assignment in project kafka by apache.

the class DescribeConsumerGroupsHandler method handleResponse.

@Override
public ApiResult<CoordinatorKey, ConsumerGroupDescription> handleResponse(Node coordinator, Set<CoordinatorKey> groupIds, AbstractResponse abstractResponse) {
    final DescribeGroupsResponse response = (DescribeGroupsResponse) abstractResponse;
    final Map<CoordinatorKey, ConsumerGroupDescription> completed = new HashMap<>();
    final Map<CoordinatorKey, Throwable> failed = new HashMap<>();
    final Set<CoordinatorKey> groupsToUnmap = new HashSet<>();
    for (DescribedGroup describedGroup : response.data().groups()) {
        CoordinatorKey groupIdKey = CoordinatorKey.byGroupId(describedGroup.groupId());
        Errors error = Errors.forCode(describedGroup.errorCode());
        if (error != Errors.NONE) {
            handleError(groupIdKey, error, failed, groupsToUnmap);
            continue;
        }
        final String protocolType = describedGroup.protocolType();
        if (protocolType.equals(ConsumerProtocol.PROTOCOL_TYPE) || protocolType.isEmpty()) {
            final List<DescribedGroupMember> members = describedGroup.members();
            final List<MemberDescription> memberDescriptions = new ArrayList<>(members.size());
            final Set<AclOperation> authorizedOperations = validAclOperations(describedGroup.authorizedOperations());
            for (DescribedGroupMember groupMember : members) {
                Set<TopicPartition> partitions = Collections.emptySet();
                if (groupMember.memberAssignment().length > 0) {
                    final Assignment assignment = ConsumerProtocol.deserializeAssignment(ByteBuffer.wrap(groupMember.memberAssignment()));
                    partitions = new HashSet<>(assignment.partitions());
                }
                memberDescriptions.add(new MemberDescription(groupMember.memberId(), Optional.ofNullable(groupMember.groupInstanceId()), groupMember.clientId(), groupMember.clientHost(), new MemberAssignment(partitions)));
            }
            final ConsumerGroupDescription consumerGroupDescription = new ConsumerGroupDescription(groupIdKey.idValue, protocolType.isEmpty(), memberDescriptions, describedGroup.protocolData(), ConsumerGroupState.parse(describedGroup.groupState()), coordinator, authorizedOperations);
            completed.put(groupIdKey, consumerGroupDescription);
        } else {
            failed.put(groupIdKey, new IllegalArgumentException(String.format("GroupId %s is not a consumer group (%s).", groupIdKey.idValue, protocolType)));
        }
    }
    return new ApiResult<>(completed, failed, new ArrayList<>(groupsToUnmap));
}