Examples with AssignmentConfigs org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs used on opensource projects

Example 6 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignWarmUpTasksIfStatefulActiveTasksBalancedOverStreamThreadsButNotOverClients.

@Test
public void shouldAssignWarmUpTasksIfStatefulActiveTasksBalancedOverStreamThreadsButNotOverClients() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_1_0, TASK_1_1);
    final Map<TaskId, Long> lagsForCaughtUpClient = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 0L));
    final Map<TaskId, Long> lagsForNotCaughtUpClient = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE));
    final ClientState caughtUpClientState = new ClientState(allTaskIds, emptySet(), lagsForCaughtUpClient, 5);
    final ClientState notCaughtUpClientState1 = new ClientState(emptySet(), emptySet(), lagsForNotCaughtUpClient, 5);
    final ClientState notCaughtUpClientState2 = new ClientState(emptySet(), emptySet(), lagsForNotCaughtUpClient, 5);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(caughtUpClientState, notCaughtUpClientState1, notCaughtUpClientState2);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, allTaskIds.size() / 3 + 1, 0, 60_000L));
    assertThat(unstable, is(true));
    assertThat(notCaughtUpClientState1.standbyTaskCount(), greaterThanOrEqualTo(allTaskIds.size() / 3));
    assertThat(notCaughtUpClientState2.standbyTaskCount(), greaterThanOrEqualTo(allTaskIds.size() / 3));
    assertValidAssignment(0, allTaskIds.size() / 3 + 1, allTaskIds, emptySet(), clientStates, new StringBuilder());
}

Example 7 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldEvenlyAssignActiveStatefulTasksIfClientsAreWarmedUpToBalanceTaskOverClients.

@Test
public void shouldEvenlyAssignActiveStatefulTasksIfClientsAreWarmedUpToBalanceTaskOverClients() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_1_0, TASK_1_1);
    final Set<TaskId> warmedUpTaskIds1 = mkSet(TASK_0_1);
    final Set<TaskId> warmedUpTaskIds2 = mkSet(TASK_1_0);
    final Map<TaskId, Long> lagsForCaughtUpClient = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 0L));
    final Map<TaskId, Long> lagsForWarmedUpClient1 = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE));
    lagsForWarmedUpClient1.put(TASK_0_1, 0L);
    final Map<TaskId, Long> lagsForWarmedUpClient2 = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE));
    lagsForWarmedUpClient2.put(TASK_1_0, 0L);
    final ClientState caughtUpClientState = new ClientState(allTaskIds, emptySet(), lagsForCaughtUpClient, 5);
    final ClientState warmedUpClientState1 = new ClientState(emptySet(), warmedUpTaskIds1, lagsForWarmedUpClient1, 5);
    final ClientState warmedUpClientState2 = new ClientState(emptySet(), warmedUpTaskIds2, lagsForWarmedUpClient2, 5);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(caughtUpClientState, warmedUpClientState1, warmedUpClientState2);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, allTaskIds.size() / 3 + 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}

Example 8 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfClientsNotIntegralDivisorOfNumberOfTasks.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfClientsNotIntegralDivisorOfNumberOfTasks() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 1);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}

Example 9 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfClientsIntegralDivisorOfNumberOfTasks.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverClientsWhereNumberOfClientsIntegralDivisorOfNumberOfTasks() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), lags, 1);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2, clientState3);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}

Example 10 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldBeStickyForActiveAndStandbyTasksWhileWarmingUp.

@Test
public void shouldBeStickyForActiveAndStandbyTasksWhileWarmingUp() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1, TASK_0_2, TASK_1_0, TASK_1_1, TASK_1_2, TASK_2_0, TASK_2_1, TASK_2_2);
    final ClientState clientState1 = new ClientState(allTaskIds, emptySet(), allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 0L)), 1);
    final ClientState clientState2 = new ClientState(emptySet(), allTaskIds, allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L)), 1);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> Long.MAX_VALUE)), 1);
    final Map<UUID, ClientState> clientStates = mkMap(mkEntry(UUID_1, clientState1), mkEntry(UUID_2, clientState2), mkEntry(UUID_3, clientState3));
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(11L, 2, 1, 60_000L));
    assertThat(clientState1, hasAssignedTasks(allTaskIds.size()));
    assertThat(clientState2, hasAssignedTasks(allTaskIds.size()));
    assertThat(clientState3, hasAssignedTasks(2));
    assertThat(unstable, is(true));
}