Examples with AssignmentConfigs org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs used on opensource projects

Example 16 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldDistributeStatelessTasksEvenlyOverClientsWithLessStreamThreadsPerClientAsTasksAndNoStatefulTasks.

@Test
public void shouldDistributeStatelessTasksEvenlyOverClientsWithLessStreamThreadsPerClientAsTasksAndNoStatefulTasks() {
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3, TASK_1_0, TASK_1_1, TASK_1_2);
    final Set<TaskId> statefulTasks = EMPTY_TASKS;
    final Set<TaskId> statelessTasks = new HashSet<>(allTasks);
    final Map<TaskId, Long> taskLags = new HashMap<>();
    final ClientState client1 = new ClientState(emptySet(), emptySet(), taskLags, 2);
    final ClientState client2 = new ClientState(emptySet(), emptySet(), taskLags, 2);
    final ClientState client3 = new ClientState(emptySet(), emptySet(), taskLags, 2);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(client1, client2, client3);
    final boolean probingRebalanceNeeded = new HighAvailabilityTaskAssignor().assign(clientStates, allTasks, statefulTasks, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertValidAssignment(0, EMPTY_TASKS, statelessTasks, clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertThat(probingRebalanceNeeded, is(false));
}

Example 17 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverClientsAndStreamThreadsWithEqualStreamThreadsPerClientAsTasks.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverClientsAndStreamThreadsWithEqualStreamThreadsPerClientAsTasks() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 9);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 9);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), lags, 9);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2, clientState3);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}

Example 18 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverUnevenlyDistributedStreamThreads.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverUnevenlyDistributedStreamThreads() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 2);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), lags, 3);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2, clientState3);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertThat(clientState1, hasActiveTasks(1));
    assertThat(clientState2, hasActiveTasks(2));
    assertThat(clientState3, hasActiveTasks(3));
    final AssignmentTestUtils.TaskSkewReport taskSkewReport = analyzeTaskAssignmentBalance(clientStates);
    if (taskSkewReport.totalSkewedTasks() == 0) {
        fail("Expected a skewed task assignment, but was: " + taskSkewReport);
    }
}

Example 19 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldDistributeStatelessTasksEvenlyOverClientsWithEqualStreamThreadsPerClientAsTasksAndNoStatefulTasks.

@Test
public void shouldDistributeStatelessTasksEvenlyOverClientsWithEqualStreamThreadsPerClientAsTasksAndNoStatefulTasks() {
    final Set<TaskId> allTasks = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_0_3, TASK_1_0, TASK_1_1, TASK_1_2);
    final Set<TaskId> statefulTasks = EMPTY_TASKS;
    final Set<TaskId> statelessTasks = new HashSet<>(allTasks);
    final Map<TaskId, Long> taskLags = new HashMap<>();
    final ClientState client1 = new ClientState(emptySet(), emptySet(), taskLags, 7);
    final ClientState client2 = new ClientState(emptySet(), emptySet(), taskLags, 7);
    final ClientState client3 = new ClientState(emptySet(), emptySet(), taskLags, 7);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(client1, client2, client3);
    final boolean probingRebalanceNeeded = new HighAvailabilityTaskAssignor().assign(clientStates, allTasks, statefulTasks, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertValidAssignment(0, EMPTY_TASKS, statelessTasks, clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertThat(probingRebalanceNeeded, is(false));
}

Example 20 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfThreadsIntegralDivisorOfNumberOfTasks.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfThreadsIntegralDivisorOfNumberOfTasks() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 3);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 3);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), lags, 3);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2, clientState3);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}