Examples with TopicPartition org.zalando.nakadi.domain.TopicPartition used on opensource projects

Example 1 with TopicPartition

use of org.zalando.nakadi.domain.TopicPartition in project nakadi by zalando.

the class MultiTimelineEventConsumer method electTopicRepositories.

private void electTopicRepositories() throws NakadiException, InvalidCursorException {
    final Map<TopicRepository, List<NakadiCursor>> newAssignment = new HashMap<>();
    borderOffsets.clear();
    // Purpose of this collection is to hold tr that definitely changed their positions and should be recreated.
    final Set<TopicPartition> actualReadPositionChanged = new HashSet<>();
    // load new topic repositories and possibly replace cursors to newer timelines.
    for (final NakadiCursor cursor : latestOffsets.values()) {
        final AtomicReference<NakadiCursor> cursorReplacement = new AtomicReference<>();
        final TopicRepository topicRepository = selectCorrectTopicRepo(cursor, cursorReplacement::set, nc -> Optional.ofNullable(nc).ifPresent(itm -> borderOffsets.put(itm.getEventTypePartition(), itm.getOffset())));
        if (!newAssignment.containsKey(topicRepository)) {
            newAssignment.put(topicRepository, new ArrayList<>());
        }
        if (cursorReplacement.get() != null) {
            actualReadPositionChanged.add(cursor.getTopicPartition());
            newAssignment.get(topicRepository).add(cursorReplacement.get());
        } else {
            newAssignment.get(topicRepository).add(cursor);
        }
    }
    final Set<TopicRepository> removedTopicRepositories = eventConsumers.keySet().stream().filter(tr -> !newAssignment.containsKey(tr)).collect(Collectors.toSet());
    // Stop and remove event consumers that are not needed anymore
    for (final TopicRepository toRemove : removedTopicRepositories) {
        stopAndRemoveConsumer(toRemove);
    }
    // Stop and remove event consumers with changed configuration
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : newAssignment.entrySet()) {
        final EventConsumer.LowLevelConsumer existingEventConsumer = eventConsumers.get(entry.getKey());
        if (null != existingEventConsumer) {
            final Set<TopicPartition> newTopicPartitions = entry.getValue().stream().map(NakadiCursor::getTopicPartition).collect(Collectors.toSet());
            final Set<TopicPartition> oldAssignment = existingEventConsumer.getAssignment();
            if (!oldAssignment.equals(newTopicPartitions) || oldAssignment.stream().anyMatch(actualReadPositionChanged::contains)) {
                stopAndRemoveConsumer(entry.getKey());
            }
        }
    }
    // Start new consumers with changed configuration.
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : newAssignment.entrySet()) {
        if (!eventConsumers.containsKey(entry.getKey())) {
            final TopicRepository repo = entry.getKey();
            LOG.info("Creating underlying consumer for client id {} and cursors {}", clientId, Arrays.deepToString(entry.getValue().toArray()));
            final EventConsumer.LowLevelConsumer consumer = repo.createEventConsumer(clientId, entry.getValue());
            eventConsumers.put(repo, consumer);
        }
    }
}