Examples with UNKNOWN_OFFSET_SUM org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo.UNKNOWN_OFFSET_SUM used on opensource projects

Example 1 with UNKNOWN_OFFSET_SUM

use of org.apache.kafka.streams.processor.internals.assignment.SubscriptionInfo.UNKNOWN_OFFSET_SUM in project kafka by apache.

the class StreamsPartitionAssignor method populateClientStatesMap.

/**
 * Builds a map from client to state, and readies each ClientState for assignment by adding any missing prev tasks
 * and computing the per-task overall lag based on the fetched end offsets for each changelog.
 *
 * @param clientStates a map from each client to its state, including offset lags. Populated by this method.
 * @param clientMetadataMap a map from each client to its full metadata
 * @param taskForPartition map from topic partition to its corresponding task
 * @param changelogTopics object that manages changelog topics
 *
 * @return whether we were able to successfully fetch the changelog end offsets and compute each client's lag
 */
private boolean populateClientStatesMap(final Map<UUID, ClientState> clientStates, final Map<UUID, ClientMetadata> clientMetadataMap, final Map<TopicPartition, TaskId> taskForPartition, final ChangelogTopics changelogTopics) {
    boolean fetchEndOffsetsSuccessful;
    Map<TaskId, Long> allTaskEndOffsetSums;
    try {
        // Make the listOffsets request first so it can  fetch the offsets for non-source changelogs
        // asynchronously while we use the blocking Consumer#committed call to fetch source-changelog offsets
        final KafkaFuture<Map<TopicPartition, ListOffsetsResultInfo>> endOffsetsFuture = fetchEndOffsetsFuture(changelogTopics.preExistingNonSourceTopicBasedPartitions(), adminClient);
        final Map<TopicPartition, Long> sourceChangelogEndOffsets = fetchCommittedOffsets(changelogTopics.preExistingSourceTopicBasedPartitions(), mainConsumerSupplier.get());
        final Map<TopicPartition, ListOffsetsResultInfo> endOffsets = ClientUtils.getEndOffsets(endOffsetsFuture);
        allTaskEndOffsetSums = computeEndOffsetSumsByTask(endOffsets, sourceChangelogEndOffsets, changelogTopics);
        fetchEndOffsetsSuccessful = true;
    } catch (final StreamsException | TimeoutException e) {
        allTaskEndOffsetSums = changelogTopics.statefulTaskIds().stream().collect(Collectors.toMap(t -> t, t -> UNKNOWN_OFFSET_SUM));
        fetchEndOffsetsSuccessful = false;
    }
    for (final Map.Entry<UUID, ClientMetadata> entry : clientMetadataMap.entrySet()) {
        final UUID uuid = entry.getKey();
        final ClientState state = entry.getValue().state;
        state.initializePrevTasks(taskForPartition, taskManager.topologyMetadata().hasNamedTopologies());
        state.computeTaskLags(uuid, allTaskEndOffsetSums);
        clientStates.put(uuid, state);
    }
    return fetchEndOffsetsSuccessful;
}