Examples with ConnectorTaskId org.apache.kafka.connect.util.ConnectorTaskId used on opensource projects

Example 26 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class IncrementalCooperativeAssignor method performTaskRevocation.

/**
 * Task revocation is based on an rough estimation of the lower average number of tasks before
 * and after new workers join the group. If no new workers join, no revocation takes place.
 * Based on this estimation, tasks are revoked until the new floor average is reached for
 * each existing worker. The revoked tasks, once assigned to the new workers will maintain
 * a balanced load among the group.
 *
 * @param activeAssignments
 * @param completeWorkerAssignment
 * @return
 */
private Map<String, ConnectorsAndTasks> performTaskRevocation(ConnectorsAndTasks activeAssignments, Collection<WorkerLoad> completeWorkerAssignment) {
    int totalActiveConnectorsNum = activeAssignments.connectors().size();
    int totalActiveTasksNum = activeAssignments.tasks().size();
    Collection<WorkerLoad> existingWorkers = completeWorkerAssignment.stream().filter(wl -> wl.size() > 0).collect(Collectors.toList());
    int existingWorkersNum = existingWorkers.size();
    int totalWorkersNum = completeWorkerAssignment.size();
    int newWorkersNum = totalWorkersNum - existingWorkersNum;
    if (log.isDebugEnabled()) {
        completeWorkerAssignment.forEach(wl -> log.debug("Per worker current load size; worker: {} connectors: {} tasks: {}", wl.worker(), wl.connectorsSize(), wl.tasksSize()));
    }
    Map<String, ConnectorsAndTasks> revoking = new HashMap<>();
    // after logging the status
    if (!(newWorkersNum > 0 && existingWorkersNum > 0)) {
        log.debug("No task revocation required; workers with existing load: {} workers with " + "no load {} total workers {}", existingWorkersNum, newWorkersNum, totalWorkersNum);
        // connectors and tasks as well
        return revoking;
    }
    log.debug("Task revocation is required; workers with existing load: {} workers with " + "no load {} total workers {}", existingWorkersNum, newWorkersNum, totalWorkersNum);
    // We have at least one worker assignment (the leader itself) so totalWorkersNum can't be 0
    log.debug("Previous rounded down (floor) average number of connectors per worker {}", totalActiveConnectorsNum / existingWorkersNum);
    int floorConnectors = totalActiveConnectorsNum / totalWorkersNum;
    int ceilConnectors = floorConnectors + ((totalActiveConnectorsNum % totalWorkersNum == 0) ? 0 : 1);
    log.debug("New average number of connectors per worker rounded down (floor) {} and rounded up (ceil) {}", floorConnectors, ceilConnectors);
    log.debug("Previous rounded down (floor) average number of tasks per worker {}", totalActiveTasksNum / existingWorkersNum);
    int floorTasks = totalActiveTasksNum / totalWorkersNum;
    int ceilTasks = floorTasks + ((totalActiveTasksNum % totalWorkersNum == 0) ? 0 : 1);
    log.debug("New average number of tasks per worker rounded down (floor) {} and rounded up (ceil) {}", floorTasks, ceilTasks);
    int numToRevoke;
    for (WorkerLoad existing : existingWorkers) {
        Iterator<String> connectors = existing.connectors().iterator();
        numToRevoke = existing.connectorsSize() - ceilConnectors;
        for (int i = existing.connectorsSize(); i > floorConnectors && numToRevoke > 0; --i, --numToRevoke) {
            ConnectorsAndTasks resources = revoking.computeIfAbsent(existing.worker(), w -> new ConnectorsAndTasks.Builder().build());
            resources.connectors().add(connectors.next());
        }
    }
    for (WorkerLoad existing : existingWorkers) {
        Iterator<ConnectorTaskId> tasks = existing.tasks().iterator();
        numToRevoke = existing.tasksSize() - ceilTasks;
        log.debug("Tasks on worker {} is higher than ceiling, so revoking {} tasks", existing, numToRevoke);
        for (int i = existing.tasksSize(); i > floorTasks && numToRevoke > 0; --i, --numToRevoke) {
            ConnectorsAndTasks resources = revoking.computeIfAbsent(existing.worker(), w -> new ConnectorsAndTasks.Builder().build());
            resources.tasks().add(tasks.next());
        }
    }
    return revoking;
}

Example 27 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class IncrementalCooperativeAssignor method assignTasks.

/**
 * Perform a round-robin assignment of tasks to workers with existing worker load. This
 * assignment tries to balance the load between workers, by assigning tasks to workers that
 * have equal load, starting with the least loaded workers.
 *
 * @param workerAssignment the current worker assignment; assigned tasks are added to this list
 * @param tasks the tasks to be assigned
 */
protected void assignTasks(List<WorkerLoad> workerAssignment, Collection<ConnectorTaskId> tasks) {
    workerAssignment.sort(WorkerLoad.taskComparator());
    WorkerLoad first = workerAssignment.get(0);
    Iterator<ConnectorTaskId> load = tasks.iterator();
    while (load.hasNext()) {
        int firstLoad = first.tasksSize();
        int upTo = IntStream.range(0, workerAssignment.size()).filter(i -> workerAssignment.get(i).tasksSize() > firstLoad).findFirst().orElse(workerAssignment.size());
        for (WorkerLoad worker : workerAssignment.subList(0, upTo)) {
            ConnectorTaskId task = load.next();
            log.debug("Assigning task {} to {}", task, worker.worker());
            worker.assign(task);
            if (!load.hasNext()) {
                break;
            }
        }
    }
}

Example 28 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class EagerAssignor method fillAssignmentsAndSerialize.

private Map<String, ByteBuffer> fillAssignmentsAndSerialize(Collection<String> members, short error, String leaderId, String leaderUrl, long maxOffset, Map<String, Collection<String>> connectorAssignments, Map<String, Collection<ConnectorTaskId>> taskAssignments) {
    Map<String, ByteBuffer> groupAssignment = new HashMap<>();
    for (String member : members) {
        Collection<String> connectors = connectorAssignments.get(member);
        if (connectors == null) {
            connectors = Collections.emptyList();
        }
        Collection<ConnectorTaskId> tasks = taskAssignments.get(member);
        if (tasks == null) {
            tasks = Collections.emptyList();
        }
        Assignment assignment = new Assignment(error, leaderId, leaderUrl, maxOffset, connectors, tasks);
        log.debug("Assignment: {} -> {}", member, assignment);
        groupAssignment.put(member, ConnectProtocol.serializeAssignment(assignment));
    }
    log.debug("Finished assignment");
    return groupAssignment;
}

Example 29 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class IncrementalCooperativeAssignor method handleLostAssignments.

// visible for testing
protected void handleLostAssignments(ConnectorsAndTasks lostAssignments, ConnectorsAndTasks newSubmissions, List<WorkerLoad> completeWorkerAssignment, Map<String, ExtendedWorkerState> memberConfigs) {
    if (lostAssignments.isEmpty()) {
        resetDelay();
        return;
    }
    final long now = time.milliseconds();
    log.debug("Found the following connectors and tasks missing from previous assignments: " + lostAssignments);
    if (scheduledRebalance <= 0 && memberConfigs.keySet().containsAll(previousMembers)) {
        log.debug("No worker seems to have departed the group during the rebalance. The " + "missing assignments that the leader is detecting are probably due to some " + "workers failing to receive the new assignments in the previous rebalance. " + "Will reassign missing tasks as new tasks");
        newSubmissions.connectors().addAll(lostAssignments.connectors());
        newSubmissions.tasks().addAll(lostAssignments.tasks());
        return;
    }
    if (scheduledRebalance > 0 && now >= scheduledRebalance) {
        // delayed rebalance expired and it's time to assign resources
        log.debug("Delayed rebalance expired. Reassigning lost tasks");
        List<WorkerLoad> candidateWorkerLoad = Collections.emptyList();
        if (!candidateWorkersForReassignment.isEmpty()) {
            candidateWorkerLoad = pickCandidateWorkerForReassignment(completeWorkerAssignment);
        }
        if (!candidateWorkerLoad.isEmpty()) {
            log.debug("Assigning lost tasks to {} candidate workers: {}", candidateWorkerLoad.size(), candidateWorkerLoad.stream().map(WorkerLoad::worker).collect(Collectors.joining(",")));
            Iterator<WorkerLoad> candidateWorkerIterator = candidateWorkerLoad.iterator();
            for (String connector : lostAssignments.connectors()) {
                // Loop over the candidate workers as many times as it takes
                if (!candidateWorkerIterator.hasNext()) {
                    candidateWorkerIterator = candidateWorkerLoad.iterator();
                }
                WorkerLoad worker = candidateWorkerIterator.next();
                log.debug("Assigning connector id {} to member {}", connector, worker.worker());
                worker.assign(connector);
            }
            candidateWorkerIterator = candidateWorkerLoad.iterator();
            for (ConnectorTaskId task : lostAssignments.tasks()) {
                if (!candidateWorkerIterator.hasNext()) {
                    candidateWorkerIterator = candidateWorkerLoad.iterator();
                }
                WorkerLoad worker = candidateWorkerIterator.next();
                log.debug("Assigning task id {} to member {}", task, worker.worker());
                worker.assign(task);
            }
        } else {
            log.debug("No single candidate worker was found to assign lost tasks. Treating lost tasks as new tasks");
            newSubmissions.connectors().addAll(lostAssignments.connectors());
            newSubmissions.tasks().addAll(lostAssignments.tasks());
        }
        resetDelay();
    } else {
        candidateWorkersForReassignment.addAll(candidateWorkersForReassignment(completeWorkerAssignment));
        if (now < scheduledRebalance) {
            // a delayed rebalance is in progress, but it's not yet time to reassign
            // unaccounted resources
            delay = calculateDelay(now);
            log.debug("Delayed rebalance in progress. Task reassignment is postponed. New computed rebalance delay: {}", delay);
        } else {
            // This means scheduledRebalance == 0
            // We could also also extract the current minimum delay from the group, to make
            // independent of consecutive leader failures, but this optimization is skipped
            // at the moment
            delay = maxDelay;
            log.debug("Resetting rebalance delay to the max: {}. scheduledRebalance: {} now: {} diff scheduledRebalance - now: {}", delay, scheduledRebalance, now, scheduledRebalance - now);
        }
        scheduledRebalance = now + delay;
    }
}

Example 30 with ConnectorTaskId

use of org.apache.kafka.connect.util.ConnectorTaskId in project kafka by apache.

the class AbstractHerderTest method testConnectorStatus.

@Test
public void testConnectorStatus() {
    ConnectorTaskId taskId = new ConnectorTaskId(connector, 0);
    AbstractHerder herder = partialMockBuilder(AbstractHerder.class).withConstructor(Worker.class, String.class, String.class, StatusBackingStore.class, ConfigBackingStore.class, ConnectorClientConfigOverridePolicy.class).withArgs(worker, workerId, kafkaClusterId, statusStore, configStore, noneConnectorClientConfigOverridePolicy).addMockedMethod("generation").createMock();
    EasyMock.expect(herder.generation()).andStubReturn(generation);
    EasyMock.expect(herder.rawConfig(connector)).andReturn(null);
    EasyMock.expect(statusStore.get(connector)).andReturn(new ConnectorStatus(connector, AbstractStatus.State.RUNNING, workerId, generation));
    EasyMock.expect(statusStore.getAll(connector)).andReturn(Collections.singletonList(new TaskStatus(taskId, AbstractStatus.State.UNASSIGNED, workerId, generation)));
    replayAll();
    ConnectorStateInfo csi = herder.connectorStatus(connector);
    PowerMock.verifyAll();
}