Examples with CircularIterator org.apache.kafka.common.utils.CircularIterator used on opensource projects

Example 1 with CircularIterator

use of org.apache.kafka.common.utils.CircularIterator in project kafka by apache.

the class WorkerCoordinator method performTaskAssignment.

private Map<String, ByteBuffer> performTaskAssignment(String leaderId, long maxOffset, Map<String, ConnectProtocol.WorkerState> memberConfigs) {
    Map<String, List<String>> connectorAssignments = new HashMap<>();
    Map<String, List<ConnectorTaskId>> taskAssignments = new HashMap<>();
    // Perform round-robin task assignment. Assign all connectors and then all tasks because assigning both the
    // connector and its tasks can lead to very uneven distribution of work in some common cases (e.g. for connectors
    // that generate only 1 task each; in a cluster of 2 or an even # of nodes, only even nodes will be assigned
    // connectors and only odd nodes will be assigned tasks, but tasks are, on average, actually more resource
    // intensive than connectors).
    List<String> connectorsSorted = sorted(configSnapshot.connectors());
    CircularIterator<String> memberIt = new CircularIterator<>(sorted(memberConfigs.keySet()));
    for (String connectorId : connectorsSorted) {
        String connectorAssignedTo = memberIt.next();
        log.trace("Assigning connector {} to {}", connectorId, connectorAssignedTo);
        List<String> memberConnectors = connectorAssignments.get(connectorAssignedTo);
        if (memberConnectors == null) {
            memberConnectors = new ArrayList<>();
            connectorAssignments.put(connectorAssignedTo, memberConnectors);
        }
        memberConnectors.add(connectorId);
    }
    for (String connectorId : connectorsSorted) {
        for (ConnectorTaskId taskId : sorted(configSnapshot.tasks(connectorId))) {
            String taskAssignedTo = memberIt.next();
            log.trace("Assigning task {} to {}", taskId, taskAssignedTo);
            List<ConnectorTaskId> memberTasks = taskAssignments.get(taskAssignedTo);
            if (memberTasks == null) {
                memberTasks = new ArrayList<>();
                taskAssignments.put(taskAssignedTo, memberTasks);
            }
            memberTasks.add(taskId);
        }
    }
    this.leaderState = new LeaderState(memberConfigs, connectorAssignments, taskAssignments);
    return fillAssignmentsAndSerialize(memberConfigs.keySet(), ConnectProtocol.Assignment.NO_ERROR, leaderId, memberConfigs.get(leaderId).url(), maxOffset, connectorAssignments, taskAssignments);
}

Example 2 with CircularIterator

use of org.apache.kafka.common.utils.CircularIterator in project kafka by apache.

the class RoundRobinAssignor method assign.

@Override
public Map<String, List<TopicPartition>> assign(Map<String, Integer> partitionsPerTopic, Map<String, List<String>> subscriptions) {
    Map<String, List<TopicPartition>> assignment = new HashMap<>();
    for (String memberId : subscriptions.keySet()) assignment.put(memberId, new ArrayList<TopicPartition>());
    CircularIterator<String> assigner = new CircularIterator<>(Utils.sorted(subscriptions.keySet()));
    for (TopicPartition partition : allPartitionsSorted(partitionsPerTopic, subscriptions)) {
        final String topic = partition.topic();
        while (!subscriptions.get(assigner.peek()).contains(topic)) assigner.next();
        assignment.get(assigner.next()).add(partition);
    }
    return assignment;
}