Examples with WorkerLoad org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad used on opensource projects

Example 6 with WorkerLoad

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWhenWorkerBounces.

@Test
public void testLostAssignmentHandlingWhenWorkerBounces() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String flakyWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(flakyWorker, 2, 2, 4, 4);
    memberConfigs.remove(flakyWorker);
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // A new worker (probably returning worker) has joined
    configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build());
    memberConfigs.put(flakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.singleton(flakyWorker), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay);
    // The new worker has still no assignments
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertTrue("Wrong assignment of lost connectors", configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).connectors().containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).tasks().containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}

Example 7 with WorkerLoad

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWhenWorkerLeavesPermanently.

@Test
public void testLostAssignmentHandlingWhenWorkerLeavesPermanently() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String removedWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(removedWorker, 2, 2, 4, 4);
    memberConfigs.remove(removedWorker);
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // No new worker has joined
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    time.sleep(rebalanceDelay);
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertTrue("Wrong assignment of lost connectors", newSubmissions.connectors().containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", newSubmissions.tasks().containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}

Example 8 with WorkerLoad

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testAssignConnectorsWhenImbalanced.

@Test
public void testAssignConnectorsWhenImbalanced() {
    List<WorkerLoad> existingAssignment = new ArrayList<>();
    existingAssignment.add(workerLoad("worker0", 0, 2, 0, 2));
    existingAssignment.add(workerLoad("worker1", 2, 3, 2, 3));
    existingAssignment.add(workerLoad("worker2", 5, 4, 5, 4));
    existingAssignment.add(emptyWorkerLoad("worker3"));
    List<String> newConnectors = newConnectors(9, 24);
    List<ConnectorTaskId> newTasks = newTasks(9, 24);
    assignor.assignConnectors(existingAssignment, newConnectors);
    assignor.assignTasks(existingAssignment, newTasks);
    for (WorkerLoad worker : existingAssignment) {
        assertEquals(6, worker.connectorsSize());
        assertEquals(6, worker.tasksSize());
    }
}

Example 9 with WorkerLoad

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves.

@Test
public void testLostAssignmentHandlingWhenWorkerBouncesBackButFinallyLeaves() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String veryFlakyWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(veryFlakyWorker, 2, 2, 4, 4);
    memberConfigs.remove(veryFlakyWorker);
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - No candidate worker has appeared yet (worker with no assignments)
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // A new worker (probably returning worker) has joined
    configuredAssignment.put(veryFlakyWorker, new WorkerLoad.Builder(veryFlakyWorker).build());
    memberConfigs.put(veryFlakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.singleton(veryFlakyWorker), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay);
    // The returning worker leaves permanently after joining briefly during the delay
    configuredAssignment.remove(veryFlakyWorker);
    memberConfigs.remove(veryFlakyWorker);
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertTrue("Wrong assignment of lost connectors", newSubmissions.connectors().containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", newSubmissions.tasks().containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}

Example 10 with WorkerLoad

use of org.apache.kafka.connect.runtime.distributed.WorkerCoordinator.WorkerLoad in project kafka by apache.

the class IncrementalCooperativeAssignorTest method testLostAssignmentHandlingWithMoreThanOneCandidates.

@Test
public void testLostAssignmentHandlingWithMoreThanOneCandidates() {
    // Customize assignor for this test case
    time = new MockTime();
    initAssignor();
    assertTrue(assignor.candidateWorkersForReassignment.isEmpty());
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    Map<String, WorkerLoad> configuredAssignment = new HashMap<>();
    configuredAssignment.put("worker0", workerLoad("worker0", 0, 2, 0, 4));
    configuredAssignment.put("worker1", workerLoad("worker1", 2, 2, 4, 4));
    configuredAssignment.put("worker2", workerLoad("worker2", 4, 2, 8, 4));
    memberConfigs = memberConfigs(leader, offset, 0, 2);
    ConnectorsAndTasks newSubmissions = new ConnectorsAndTasks.Builder().build();
    // No lost assignments
    assignor.handleLostAssignments(new ConnectorsAndTasks.Builder().build(), newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    String flakyWorker = "worker1";
    WorkerLoad lostLoad = workerLoad(flakyWorker, 2, 2, 4, 4);
    memberConfigs.remove(flakyWorker);
    String newWorker = "worker3";
    ConnectorsAndTasks lostAssignments = new ConnectorsAndTasks.Builder().withCopies(lostLoad.connectors(), lostLoad.tasks()).build();
    // Lost assignments detected - A new worker also has joined that is not the returning worker
    configuredAssignment.put(newWorker, new WorkerLoad.Builder(newWorker).build());
    memberConfigs.put(newWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    assertThat("Wrong set of workers for reassignments", Collections.singleton(newWorker), is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay / 2);
    rebalanceDelay /= 2;
    // Now two new workers have joined
    configuredAssignment.put(flakyWorker, new WorkerLoad.Builder(flakyWorker).build());
    memberConfigs.put(flakyWorker, new ExtendedWorkerState(leaderUrl, offset, null));
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    Set<String> expectedWorkers = new HashSet<>();
    expectedWorkers.addAll(Arrays.asList(newWorker, flakyWorker));
    assertThat("Wrong set of workers for reassignments", expectedWorkers, is(assignor.candidateWorkersForReassignment));
    assertEquals(time.milliseconds() + rebalanceDelay, assignor.scheduledRebalance);
    assertEquals(rebalanceDelay, assignor.delay);
    assignor.previousMembers = new HashSet<>(memberConfigs.keySet());
    time.sleep(rebalanceDelay);
    // The new workers have new assignments, other than the lost ones
    configuredAssignment.put(flakyWorker, workerLoad(flakyWorker, 6, 2, 8, 4));
    configuredAssignment.put(newWorker, workerLoad(newWorker, 8, 2, 12, 4));
    // we don't reflect these new assignments in memberConfigs currently because they are not
    // used in handleLostAssignments method
    assignor.handleLostAssignments(lostAssignments, newSubmissions, new ArrayList<>(configuredAssignment.values()), memberConfigs);
    // both the newWorkers would need to be considered for re assignment of connectors and tasks
    List<String> listOfConnectorsInLast2Workers = new ArrayList<>();
    listOfConnectorsInLast2Workers.addAll(configuredAssignment.getOrDefault(newWorker, new WorkerLoad.Builder(flakyWorker).build()).connectors());
    listOfConnectorsInLast2Workers.addAll(configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).connectors());
    List<ConnectorTaskId> listOfTasksInLast2Workers = new ArrayList<>();
    listOfTasksInLast2Workers.addAll(configuredAssignment.getOrDefault(newWorker, new WorkerLoad.Builder(flakyWorker).build()).tasks());
    listOfTasksInLast2Workers.addAll(configuredAssignment.getOrDefault(flakyWorker, new WorkerLoad.Builder(flakyWorker).build()).tasks());
    assertTrue("Wrong assignment of lost connectors", listOfConnectorsInLast2Workers.containsAll(lostAssignments.connectors()));
    assertTrue("Wrong assignment of lost tasks", listOfTasksInLast2Workers.containsAll(lostAssignments.tasks()));
    assertThat("Wrong set of workers for reassignments", Collections.emptySet(), is(assignor.candidateWorkersForReassignment));
    assertEquals(0, assignor.scheduledRebalance);
    assertEquals(0, assignor.delay);
}