Examples with TaskAssignor org.apache.kafka.streams.processor.internals.assignment.TaskAssignor used on opensource projects

Example 1 with TaskAssignor

use of org.apache.kafka.streams.processor.internals.assignment.TaskAssignor in project kafka by apache.

the class StreamsPartitionAssignor method assignTasksToClients.

/**
 * Assigns a set of tasks to each client (Streams instance) using the configured task assignor, and also
 * populate the stateful tasks that have been assigned to the clients
 * @return true if a probing rebalance should be triggered
 */
private boolean assignTasksToClients(final Cluster fullMetadata, final Set<String> allSourceTopics, final Map<Subtopology, TopicsInfo> topicGroups, final Map<UUID, ClientMetadata> clientMetadataMap, final Map<TaskId, Set<TopicPartition>> partitionsForTask, final Set<TaskId> statefulTasks) {
    if (!statefulTasks.isEmpty()) {
        throw new TaskAssignmentException("The stateful tasks should not be populated before assigning tasks to clients");
    }
    final Map<TopicPartition, TaskId> taskForPartition = new HashMap<>();
    final Map<Subtopology, Set<TaskId>> tasksForTopicGroup = new HashMap<>();
    populateTasksForMaps(taskForPartition, tasksForTopicGroup, allSourceTopics, partitionsForTask, fullMetadata);
    final ChangelogTopics changelogTopics = new ChangelogTopics(internalTopicManager, topicGroups, tasksForTopicGroup, logPrefix);
    changelogTopics.setup();
    final Map<UUID, ClientState> clientStates = new HashMap<>();
    final boolean lagComputationSuccessful = populateClientStatesMap(clientStates, clientMetadataMap, taskForPartition, changelogTopics);
    log.info("All members participating in this rebalance: \n{}.", clientStates.entrySet().stream().map(entry -> entry.getKey() + ": " + entry.getValue().consumers()).collect(Collectors.joining(Utils.NL)));
    final Set<TaskId> allTasks = partitionsForTask.keySet();
    statefulTasks.addAll(changelogTopics.statefulTaskIds());
    log.debug("Assigning tasks {} including stateful {} to clients {} with number of replicas {}", allTasks, statefulTasks, clientStates, numStandbyReplicas());
    final TaskAssignor taskAssignor = createTaskAssignor(lagComputationSuccessful);
    final boolean probingRebalanceNeeded = taskAssignor.assign(clientStates, allTasks, statefulTasks, assignmentConfigs);
    log.info("Assigned tasks {} including stateful {} to clients as: \n{}.", allTasks, statefulTasks, clientStates.entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue().currentAssignment()).collect(Collectors.joining(Utils.NL)));
    return probingRebalanceNeeded;
}

Example 2 with TaskAssignor

use of org.apache.kafka.streams.processor.internals.assignment.TaskAssignor in project kafka by apache.

the class TaskAssignorIntegrationTest method shouldProperlyConfigureTheAssignor.

@SuppressWarnings("unchecked")
@Test
public void shouldProperlyConfigureTheAssignor() throws NoSuchFieldException, IllegalAccessException {
    // This test uses reflection to check and make sure that all the expected configurations really
    // make it all the way to configure the task assignor. There's no other use case for being able
    // to extract all these fields, so reflection is a good choice until we find that the maintenance
    // burden is too high.
    // 
    // Also note that this is an integration test because so many components have to come together to
    // ensure these configurations wind up where they belong, and any number of future code changes
    // could break this change.
    final String testId = safeUniqueTestName(getClass(), testName);
    final String appId = "appId_" + testId;
    final String inputTopic = "input" + testId;
    IntegrationTestUtils.cleanStateBeforeTest(CLUSTER, inputTopic);
    // Maybe I'm paranoid, but I don't want the compiler deciding that my lambdas are equal to the identity
    // function and defeating my identity check
    final AtomicInteger compilerDefeatingReference = new AtomicInteger(0);
    // the implementation doesn't matter, we're just going to verify the reference.
    final AssignmentListener configuredAssignmentListener = stable -> compilerDefeatingReference.incrementAndGet();
    final Properties properties = mkObjectProperties(mkMap(mkEntry(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, CLUSTER.bootstrapServers()), mkEntry(StreamsConfig.APPLICATION_ID_CONFIG, appId), mkEntry(StreamsConfig.STATE_DIR_CONFIG, TestUtils.tempDirectory().getPath()), mkEntry(StreamsConfig.NUM_STANDBY_REPLICAS_CONFIG, "5"), mkEntry(StreamsConfig.ACCEPTABLE_RECOVERY_LAG_CONFIG, "6"), mkEntry(StreamsConfig.MAX_WARMUP_REPLICAS_CONFIG, "7"), mkEntry(StreamsConfig.PROBING_REBALANCE_INTERVAL_MS_CONFIG, "480000"), mkEntry(StreamsConfig.InternalConfig.ASSIGNMENT_LISTENER, configuredAssignmentListener), mkEntry(StreamsConfig.InternalConfig.INTERNAL_TASK_ASSIGNOR_CLASS, MyTaskAssignor.class.getName())));
    final StreamsBuilder builder = new StreamsBuilder();
    builder.stream(inputTopic);
    try (final KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), properties)) {
        kafkaStreams.start();
        final Field threads = KafkaStreams.class.getDeclaredField("threads");
        threads.setAccessible(true);
        final List<StreamThread> streamThreads = (List<StreamThread>) threads.get(kafkaStreams);
        final StreamThread streamThread = streamThreads.get(0);
        final Field mainConsumer = StreamThread.class.getDeclaredField("mainConsumer");
        mainConsumer.setAccessible(true);
        final KafkaConsumer<?, ?> consumer = (KafkaConsumer<?, ?>) mainConsumer.get(streamThread);
        final Field assignors = KafkaConsumer.class.getDeclaredField("assignors");
        assignors.setAccessible(true);
        final List<ConsumerPartitionAssignor> consumerPartitionAssignors = (List<ConsumerPartitionAssignor>) assignors.get(consumer);
        final StreamsPartitionAssignor streamsPartitionAssignor = (StreamsPartitionAssignor) consumerPartitionAssignors.get(0);
        final Field assignmentConfigs = StreamsPartitionAssignor.class.getDeclaredField("assignmentConfigs");
        assignmentConfigs.setAccessible(true);
        final AssignorConfiguration.AssignmentConfigs configs = (AssignorConfiguration.AssignmentConfigs) assignmentConfigs.get(streamsPartitionAssignor);
        final Field assignmentListenerField = StreamsPartitionAssignor.class.getDeclaredField("assignmentListener");
        assignmentListenerField.setAccessible(true);
        final AssignmentListener actualAssignmentListener = (AssignmentListener) assignmentListenerField.get(streamsPartitionAssignor);
        final Field taskAssignorSupplierField = StreamsPartitionAssignor.class.getDeclaredField("taskAssignorSupplier");
        taskAssignorSupplierField.setAccessible(true);
        final Supplier<TaskAssignor> taskAssignorSupplier = (Supplier<TaskAssignor>) taskAssignorSupplierField.get(streamsPartitionAssignor);
        final TaskAssignor taskAssignor = taskAssignorSupplier.get();
        assertThat(configs.numStandbyReplicas, is(5));
        assertThat(configs.acceptableRecoveryLag, is(6L));
        assertThat(configs.maxWarmupReplicas, is(7));
        assertThat(configs.probingRebalanceIntervalMs, is(480000L));
        assertThat(actualAssignmentListener, sameInstance(configuredAssignmentListener));
        assertThat(taskAssignor, instanceOf(MyTaskAssignor.class));
    }
}