Examples with ConnectorsAndTasks org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks used on opensource projects

Example 6 with ConnectorsAndTasks

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks in project kafka by apache.

the class IncrementalCooperativeAssignor method computeDuplicatedAssignments.

private Map<String, ConnectorsAndTasks> computeDuplicatedAssignments(Map<String, ExtendedWorkerState> memberConfigs, Map<String, Collection<String>> connectorAssignments, Map<String, Collection<ConnectorTaskId>> taskAssignment) {
    ConnectorsAndTasks duplicatedAssignments = duplicatedAssignments(memberConfigs);
    log.debug("Duplicated assignments: {}", duplicatedAssignments);
    Map<String, ConnectorsAndTasks> toRevoke = new HashMap<>();
    if (!duplicatedAssignments.connectors().isEmpty()) {
        connectorAssignments.entrySet().stream().forEach(entry -> {
            Set<String> duplicatedConnectors = new HashSet<>(duplicatedAssignments.connectors());
            duplicatedConnectors.retainAll(entry.getValue());
            if (!duplicatedConnectors.isEmpty()) {
                toRevoke.computeIfAbsent(entry.getKey(), v -> new ConnectorsAndTasks.Builder().build()).connectors().addAll(duplicatedConnectors);
            }
        });
    }
    if (!duplicatedAssignments.tasks().isEmpty()) {
        taskAssignment.entrySet().stream().forEach(entry -> {
            Set<ConnectorTaskId> duplicatedTasks = new HashSet<>(duplicatedAssignments.tasks());
            duplicatedTasks.retainAll(entry.getValue());
            if (!duplicatedTasks.isEmpty()) {
                toRevoke.computeIfAbsent(entry.getKey(), v -> new ConnectorsAndTasks.Builder().build()).tasks().addAll(duplicatedTasks);
            }
        });
    }
    return toRevoke;
}

Example 7 with ConnectorsAndTasks

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWhenWorkerLeavesPermanently.

@Test
public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String removedWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(removedWorker, 2, 2, 4, 4);
    memberConfigs.remove(removedWorker);
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // No new worker has joined
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    time.sleep(rebalanceDelay);
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertTrue("Wrong assignment of lost connectors", newSubmissions.connectors().containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", newSubmissions.tasks().containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}

Example 8 with ConnectorsAndTasks

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves.

@Test
public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String veryFlakyWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(veryFlakyWorker, 2, 2, 4, 4);
    memberConfigs.remove(veryFlakyWorker);
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // A new worker (probably returning worker) has joined
    configuredAssignment.put(veryFlakyWorker, new WorkerLoad.Builder(veryFlakyWorker).build());
    memberConfigs.put(veryFlakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.singleton(veryFlakyWorker), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay);
    // The returning worker leaves permanently after joining briefly during the delay
    configuredAssignment.remove(veryFlakyWorker);
    memberConfigs.remove(veryFlakyWorker);
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertTrue("Wrong assignment of lost connectors", newSubmissions.connectors().containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", newSubmissions.tasks().containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}

Example 9 with ConnectorsAndTasks

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.ConnectorsAndTasks in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWithMoreThanOneCandidates.

@Test
public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String flakyWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(flakyWorker, 2, 2, 4, 4);
    memberConfigs.remove(flakyWorker);
    String newWorker = "worker3";
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - A new worker also has joined that is not the returning worker
    configuredAssignment.put(newWorker, new WorkerLoad.Builder(newWorker).build());
    memberConfigs.put(newWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.singleton(newWorker), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // Now two new workers have joined
    configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build());
    memberConfigs.put(flakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    Set<String> expectedWorkers = new HashSet<>();
    expectedWorkers.addAll(Arrays.asList(newWorker, flakyWorker));
    assertThat("Wrong set of workers for reassignments", expectedWorkers, is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay);
    // The new workers have new assignments, other than the lost ones
    configuredAssignment.put(flakyWorker, workerLoad(flakyWorker, 6, 2, 8, 4));
    configuredAssignment.put(newWorker, workerLoad(newWorker, 8, 2, 12, 4));
    // we don't reflect these new assignments in memberConfigs currently because they are not
    // used in handleLostAssignments method
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    // both the newWorkers would need to be considered for re assignment of connectors and tasks
    List<String> listOfConnectorsInLast2Workers = new ArrayList<>();
    listOfConnectorsInLast2Workers.addAll(configuredAssignment.getOrDefault(newWorker, new WorkerLoad.Builder(flakyWorker).build()).connectors());
    listOfConnectorsInLast2Workers.addAll(configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).connectors());
    List<ConnectorTaskId> listOfTasksInLast2Workers = new ArrayList<>();
    listOfTasksInLast2Workers.addAll(configuredAssignment.getOrDefault(newWorker, new WorkerLoad.Builder(flakyWorker).build()).tasks());
    listOfTasksInLast2Workers.addAll(configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).tasks());
    assertTrue("Wrong assignment of lost connectors", listOfConnectorsInLast2Workers.containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", listOfTasksInLast2Workers.containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}