Examples with AssignmentConfigs org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs used on opensource projects

Example 11 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldSkipWarmupsWhenAcceptableLagIsMax.

@Test
public void shouldSkipWarmupsWhenAcceptableLagIsMax() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final ClientState clientState1 = new ClientState(allTaskIds, emptySet(), allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 0L)), 1);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE)), 1);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE)), 1);
    final Map<UUID, ClientState> clientStates = mkMap(mkEntry(UUID_1, clientState1), mkEntry(UUID_2, clientState2), mkEntry(UUID_3, clientState3));
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(Long.MAX_VALUE, 1, 1, 60_000L));
    assertThat(clientState1, hasAssignedTasks(6));
    assertThat(clientState2, hasAssignedTasks(6));
    assertThat(clientState3, hasAssignedTasks(6));
    assertThat(unstable, is(false));
}

Example 12 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldDistributeStatelessTasksEvenlyWithPreviousAssignmentAndNoStatefulTasks.

@Test
public void shouldDistributeStatelessTasksEvenlyWithPreviousAssignmentAndNoStatefulTasks() {
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3, TASK_1_0, TASK_1_1, TASK_1_2);
    final Set<TaskId> statefulTasks = EMPTY_TASKS;
    final Set<TaskId> statelessTasks = new HashSet<>(allTasks);
    final Map<TaskId, Long> taskLags = new HashMap<>();
    final ClientState client1 = new ClientState(statelessTasks, emptySet(), taskLags, 3);
    final ClientState client2 = new ClientState(emptySet(), emptySet(), taskLags, 3);
    final ClientState client3 = new ClientState(emptySet(), emptySet(), taskLags, 3);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(client1, client2, client3);
    final boolean probingRebalanceNeeded = new HighAvailabilityTaskAssignor().assign(clientStates, allTasks, statefulTasks, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertValidAssignment(0, EMPTY_TASKS, statelessTasks, clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertThat(probingRebalanceNeeded, is(false));
}

Example 13 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldNotAssignMoreThanMaxWarmupReplicas.

@Test
public void shouldNotAssignMoreThanMaxWarmupReplicas() {
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3);
    final Set<TaskId> statefulTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3);
    final ClientState client1 = getMockClientWithPreviousCaughtUpTasks(mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3), statefulTasks);
    final ClientState client2 = getMockClientWithPreviousCaughtUpTasks(EMPTY_TASKS, statefulTasks);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(client1, client2);
    final boolean probingRebalanceNeeded = new HighAvailabilityTaskAssignor().assign(clientStates, allTasks, statefulTasks, new AssignmentConfigs(/*acceptableRecoveryLag*/
    100L, /*maxWarmupReplicas*/
    1, /*numStandbyReplicas*/
    0, /*probingRebalanceIntervalMs*/
    60 * 1000L));
    assertThat(client1.activeTasks(), equalTo(mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3)));
    assertThat(client2.standbyTaskCount(), equalTo(1));
    assertHasNoStandbyTasks(client1);
    assertHasNoActiveTasks(client2);
    assertThat(probingRebalanceNeeded, is(true));
}

Example 14 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldNotAssignWarmupAndStandbyToTheSameClient.

@Test
public void shouldNotAssignWarmupAndStandbyToTheSameClient() {
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3);
    final Set<TaskId> statefulTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3);
    final ClientState client1 = getMockClientWithPreviousCaughtUpTasks(mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3), statefulTasks);
    final ClientState client2 = getMockClientWithPreviousCaughtUpTasks(EMPTY_TASKS, statefulTasks);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(client1, client2);
    final boolean probingRebalanceNeeded = new HighAvailabilityTaskAssignor().assign(clientStates, allTasks, statefulTasks, new AssignmentConfigs(/*acceptableRecoveryLag*/
    100L, /*maxWarmupReplicas*/
    1, /*numStandbyReplicas*/
    1, /*probingRebalanceIntervalMs*/
    60 * 1000L));
    assertThat(client1.activeTasks(), equalTo(mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3)));
    assertThat(client2.standbyTasks(), equalTo(mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3)));
    assertHasNoStandbyTasks(client1);
    assertHasNoActiveTasks(client2);
    assertThat(probingRebalanceNeeded, is(true));
}

Example 15 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverStreamThreadsButBestEffortOverClients.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverStreamThreadsButBestEffortOverClients() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 6);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 3);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertThat(clientState1, hasActiveTasks(6));
    assertThat(clientState2, hasActiveTasks(3));
}