Examples with AssignmentConfigs org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs used on opensource projects

Example 1 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class HighAvailabilityTaskAssignorTest method shouldAssignActiveStatefulTasksEvenlyOverClientsWithMoreClientsThanTasks.

@Test
public void shouldAssignActiveStatefulTasksEvenlyOverClientsWithMoreClientsThanTasks() {
    final Set<TaskId> allTaskIds = mkSet(TASK_0_0, TASK_0_1);
    final Map<TaskId, Long> lags = allTaskIds.stream().collect(Collectors.toMap(k -> k, k -> 10L));
    final ClientState clientState1 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState2 = new ClientState(emptySet(), emptySet(), lags, 1);
    final ClientState clientState3 = new ClientState(emptySet(), emptySet(), lags, 1);
    final Map<UUID, ClientState> clientStates = getClientStatesMap(clientState1, clientState2, clientState3);
    final boolean unstable = new HighAvailabilityTaskAssignor().assign(clientStates, allTaskIds, allTaskIds, new AssignmentConfigs(0L, 1, 0, 60_000L));
    assertThat(unstable, is(false));
    assertValidAssignment(0, allTaskIds, emptySet(), clientStates, new StringBuilder());
    assertBalancedActiveAssignment(clientStates, new StringBuilder());
    assertBalancedStatefulAssignment(allTaskIds, clientStates, new StringBuilder());
    assertBalancedTasks(clientStates);
}

Example 2 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class TaskAssignorConvergenceTest method staticAssignmentShouldConvergeWithTheFirstAssignment.

@Test
public void staticAssignmentShouldConvergeWithTheFirstAssignment() {
    final AssignmentConfigs configs = new AssignmentConfigs(100L, 2, 0, 60_000L);
    final Harness harness = Harness.initializeCluster(1, 1, 1, () -> 1);
    testForConvergence(harness, configs, 1);
    verifyValidAssignment(0, harness);
    verifyBalancedAssignment(harness);
}

Example 3 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class TaskAssignorConvergenceTest method runRandomizedScenario.

private static void runRandomizedScenario(final long seed) {
    Harness harness = null;
    try {
        final Random prng = new Random(seed);
        // These are all rand(limit)+1 because we need them to be at least 1 and the upper bound is exclusive
        final int initialClusterSize = prng.nextInt(10) + 1;
        final int numStatelessTasks = prng.nextInt(10) + 1;
        final int numStatefulTasks = prng.nextInt(10) + 1;
        final int maxWarmupReplicas = prng.nextInt(numStatefulTasks) + 1;
        // This one is rand(limit+1) because we _want_ to test zero and the upper bound is exclusive
        final int numStandbyReplicas = prng.nextInt(initialClusterSize + 1);
        final int numberOfEvents = prng.nextInt(10) + 1;
        final AssignmentConfigs configs = new AssignmentConfigs(100L, maxWarmupReplicas, numStandbyReplicas, 60_000L);
        harness = Harness.initializeCluster(numStatelessTasks, numStatefulTasks, initialClusterSize, () -> prng.nextInt(10) + 1);
        testForConvergence(harness, configs, 1);
        verifyValidAssignment(numStandbyReplicas, harness);
        verifyBalancedAssignment(harness);
        for (int i = 0; i < numberOfEvents; i++) {
            final int event = prng.nextInt(2);
            switch(event) {
                case 0:
                    harness.dropRandomNodes(prng.nextInt(initialClusterSize), prng);
                    break;
                case 1:
                    harness.addOrResurrectNodesRandomly(prng, initialClusterSize);
                    break;
                default:
                    throw new IllegalStateException("Unexpected event: " + event);
            }
            if (!harness.clientStates.isEmpty()) {
                testForConvergence(harness, configs, 2 * (numStatefulTasks + numStatefulTasks * numStandbyReplicas));
                verifyValidAssignment(numStandbyReplicas, harness);
                verifyBalancedAssignment(harness);
            }
        }
    } catch (final AssertionError t) {
        throw new AssertionError("Assertion failed in randomized test. Reproduce with: `runRandomizedScenario(" + seed + ")`.", t);
    } catch (final Throwable t) {
        final StringBuilder builder = new StringBuilder().append("Exception in randomized scenario. Reproduce with: `runRandomizedScenario(").append(seed).append(")`. ");
        if (harness != null) {
            builder.append(harness.history);
        }
        throw new AssertionError(builder.toString(), t);
    }
}

Example 4 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class TaskAssignorConvergenceTest method assignmentShouldConvergeAfterAddingNode.

@Test
public void assignmentShouldConvergeAfterAddingNode() {
    final int numStatelessTasks = 7;
    final int numStatefulTasks = 11;
    final int maxWarmupReplicas = 2;
    final int numStandbyReplicas = 0;
    final AssignmentConfigs configs = new AssignmentConfigs(100L, maxWarmupReplicas, numStandbyReplicas, 60_000L);
    final Harness harness = Harness.initializeCluster(numStatelessTasks, numStatefulTasks, 1, () -> 5);
    testForConvergence(harness, configs, 1);
    harness.addNode();
    // we expect convergence to involve moving each task at most once, and we can move "maxWarmupReplicas" number
    // of tasks at once, hence the iteration limit
    testForConvergence(harness, configs, numStatefulTasks / maxWarmupReplicas + 1);
    verifyValidAssignment(numStandbyReplicas, harness);
    verifyBalancedAssignment(harness);
}

Example 5 with AssignmentConfigs

use of org.apache.kafka.streams.processor.internals.assignment.AssignorConfiguration.AssignmentConfigs in project kafka by apache.

the class TaskAssignorConvergenceTest method droppingNodesShouldConverge.

@Test
public void droppingNodesShouldConverge() {
    final int numStatelessTasks = 11;
    final int numStatefulTasks = 13;
    final int maxWarmupReplicas = 2;
    final int numStandbyReplicas = 0;
    final AssignmentConfigs configs = new AssignmentConfigs(100L, maxWarmupReplicas, numStandbyReplicas, 60_000L);
    final Harness harness = Harness.initializeCluster(numStatelessTasks, numStatefulTasks, 7, () -> 5);
    testForConvergence(harness, configs, 1);
    harness.dropNode();
    // This time, we allow one extra iteration because the
    // first stateful task needs to get shuffled back to the first node
    testForConvergence(harness, configs, numStatefulTasks / maxWarmupReplicas + 2);
    verifyValidAssignment(numStandbyReplicas, harness);
    verifyBalancedAssignment(harness);
}