Examples with PartitionStatistics org.zalando.nakadi.domain.PartitionStatistics used on opensource projects

Example 6 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class KafkaTopicRepository method loadTopicStatistics.

@Override
public List<PartitionStatistics> loadTopicStatistics(final Collection<Timeline> timelines) throws ServiceUnavailableException {
    try (Consumer<byte[], byte[]> consumer = kafkaFactory.getConsumer()) {
        final Map<TopicPartition, Timeline> backMap = new HashMap<>();
        for (final Timeline timeline : timelines) {
            consumer.partitionsFor(timeline.getTopic()).stream().map(p -> new TopicPartition(p.topic(), p.partition())).forEach(tp -> backMap.put(tp, timeline));
        }
        final List<TopicPartition> kafkaTPs = new ArrayList<>(backMap.keySet());
        consumer.assign(kafkaTPs);
        consumer.seekToBeginning(kafkaTPs);
        final long[] begins = kafkaTPs.stream().mapToLong(consumer::position).toArray();
        consumer.seekToEnd(kafkaTPs);
        final long[] ends = kafkaTPs.stream().mapToLong(consumer::position).toArray();
        return IntStream.range(0, kafkaTPs.size()).mapToObj(i -> new KafkaPartitionStatistics(backMap.get(kafkaTPs.get(i)), kafkaTPs.get(i).partition(), begins[i], ends[i] - 1)).collect(toList());
    } catch (final Exception e) {
        throw new ServiceUnavailableException("Error occurred when fetching partitions offsets", e);
    }
}

Example 7 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class KafkaTopicRepository method loadPartitionStatistics.

@Override
public List<Optional<PartitionStatistics>> loadPartitionStatistics(final Collection<TimelinePartition> partitions) throws ServiceUnavailableException {
    final Map<String, Set<String>> topicToPartitions = partitions.stream().collect(Collectors.groupingBy(tp -> tp.getTimeline().getTopic(), Collectors.mapping(TimelinePartition::getPartition, Collectors.toSet())));
    try (Consumer<byte[], byte[]> consumer = kafkaFactory.getConsumer()) {
        final List<PartitionInfo> allKafkaPartitions = topicToPartitions.keySet().stream().map(consumer::partitionsFor).flatMap(Collection::stream).collect(Collectors.toList());
        final List<TopicPartition> partitionsToQuery = allKafkaPartitions.stream().filter(pi -> topicToPartitions.get(pi.topic()).contains(KafkaCursor.toNakadiPartition(pi.partition()))).map(pi -> new TopicPartition(pi.topic(), pi.partition())).collect(Collectors.toList());
        consumer.assign(partitionsToQuery);
        consumer.seekToBeginning(partitionsToQuery);
        final List<Long> begins = partitionsToQuery.stream().map(consumer::position).collect(toList());
        consumer.seekToEnd(partitionsToQuery);
        final List<Long> ends = partitionsToQuery.stream().map(consumer::position).collect(toList());
        final List<Optional<PartitionStatistics>> result = new ArrayList<>(partitions.size());
        for (final TimelinePartition tap : partitions) {
            // Now search for an index.
            final Optional<PartitionStatistics> itemResult = IntStream.range(0, partitionsToQuery.size()).filter(i -> {
                final TopicPartition info = partitionsToQuery.get(i);
                return info.topic().equals(tap.getTimeline().getTopic()) && info.partition() == KafkaCursor.toKafkaPartition(tap.getPartition());
            }).mapToObj(indexFound -> (PartitionStatistics) new KafkaPartitionStatistics(tap.getTimeline(), partitionsToQuery.get(indexFound).partition(), begins.get(indexFound), ends.get(indexFound) - 1L)).findAny();
            result.add(itemResult);
        }
        return result;
    } catch (final Exception e) {
        throw new ServiceUnavailableException("Error occurred when fetching partitions offsets", e);
    }
}

Example 8 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class VersionZeroConverter method convertBatched.

public List<NakadiCursor> convertBatched(final List<SubscriptionCursorWithoutToken> cursors) throws InvalidCursorException, InternalNakadiException, NoSuchEventTypeException, ServiceUnavailableException {
    final NakadiCursor[] result = new NakadiCursor[cursors.size()];
    for (int idx = 0; idx < cursors.size(); ++idx) {
        final SubscriptionCursorWithoutToken cursor = cursors.get(idx);
        if (Cursor.BEFORE_OLDEST_OFFSET.equalsIgnoreCase(cursor.getOffset())) {
            // Preform begin checks afterwards to optimize calls
            continue;
        }
        if (!NUMBERS_ONLY_PATTERN.matcher(cursor.getOffset()).matches()) {
            throw new InvalidCursorException(CursorError.INVALID_OFFSET, cursor);
        }
    }
    // now it is time for massive convert.
    final LinkedHashMap<SubscriptionCursorWithoutToken, NakadiCursor> beginsToConvert = new LinkedHashMap<>();
    final Map<SubscriptionCursorWithoutToken, Timeline> cursorTimelines = new HashMap<>();
    final Map<TopicRepository, List<SubscriptionCursorWithoutToken>> repos = new HashMap<>();
    for (int i = 0; i < result.length; ++i) {
        if (null == result[i]) {
            // cursor requires database hit
            final SubscriptionCursorWithoutToken cursor = cursors.get(i);
            final Timeline timeline = timelineService.getActiveTimelinesOrdered(cursor.getEventType()).get(0);
            final TopicRepository topicRepo = timelineService.getTopicRepository(timeline);
            beginsToConvert.put(cursor, null);
            cursorTimelines.put(cursor, timeline);
            repos.computeIfAbsent(topicRepo, k -> new ArrayList<>()).add(cursor);
        }
    }
    for (final Map.Entry<TopicRepository, List<SubscriptionCursorWithoutToken>> entry : repos.entrySet()) {
        final List<Optional<PartitionStatistics>> stats = entry.getKey().loadPartitionStatistics(entry.getValue().stream().map(scwt -> new TopicRepository.TimelinePartition(cursorTimelines.get(scwt), scwt.getPartition())).collect(Collectors.toList()));
        for (int idx = 0; idx < entry.getValue().size(); ++idx) {
            // Reinsert doesn't change the order
            beginsToConvert.put(entry.getValue().get(idx), stats.get(idx).orElseThrow(() -> new InvalidCursorException(PARTITION_NOT_FOUND)).getBeforeFirst());
        }
    }
    final Iterator<NakadiCursor> missingBegins = beginsToConvert.values().iterator();
    return Stream.of(result).map(it -> null == it ? missingBegins.next() : it).collect(Collectors.toList());
}

Example 9 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class EventStreamController method getStreamingStart.

@VisibleForTesting
List<NakadiCursor> getStreamingStart(final EventType eventType, final String cursorsStr) throws UnparseableCursorException, ServiceUnavailableException, InvalidCursorException, InternalNakadiException, NoSuchEventTypeException {
    List<Cursor> cursors = null;
    if (cursorsStr != null) {
        try {
            cursors = jsonMapper.readValue(cursorsStr, new TypeReference<ArrayList<Cursor>>() {
            });
        } catch (final IOException ex) {
            throw new UnparseableCursorException("incorrect syntax of X-nakadi-cursors header", ex, cursorsStr);
        }
        // Unfortunately, In order to have consistent exception checking, one can not just call validator
        for (final Cursor cursor : cursors) {
            if (null == cursor.getPartition()) {
                throw new InvalidCursorException(CursorError.NULL_PARTITION, cursor);
            } else if (null == cursor.getOffset()) {
                throw new InvalidCursorException(CursorError.NULL_OFFSET, cursor);
            }
        }
    }
    final Timeline latestTimeline = timelineService.getActiveTimeline(eventType);
    if (null != cursors) {
        if (cursors.isEmpty()) {
            throw new InvalidCursorException(CursorError.INVALID_FORMAT);
        }
        final List<NakadiCursor> result = new ArrayList<>();
        for (final Cursor c : cursors) {
            result.add(cursorConverter.convert(eventType.getName(), c));
        }
        for (final NakadiCursor c : result) {
            if (c.getTimeline().isDeleted()) {
                throw new InvalidCursorException(CursorError.UNAVAILABLE, c);
            }
        }
        final Map<Storage, List<NakadiCursor>> groupedCursors = result.stream().collect(Collectors.groupingBy(c -> c.getTimeline().getStorage()));
        for (final Map.Entry<Storage, List<NakadiCursor>> entry : groupedCursors.entrySet()) {
            timelineService.getTopicRepository(entry.getKey()).validateReadCursors(entry.getValue());
        }
        return result;
    } else {
        final TopicRepository latestTopicRepository = timelineService.getTopicRepository(latestTimeline);
        // if no cursors provided - read from the newest available events
        return latestTopicRepository.loadTopicStatistics(Collections.singletonList(latestTimeline)).stream().map(PartitionStatistics::getLast).collect(Collectors.toList());
    }
}

Example 10 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class PartitionsController method getTopicPartition.

private EventTypePartitionView getTopicPartition(final String eventTypeName, final String partition) throws InternalNakadiException, NoSuchEventTypeException, ServiceUnavailableException {
    final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventTypeName);
    final Optional<PartitionStatistics> firstStats = timelineService.getTopicRepository(timelines.get(0)).loadPartitionStatistics(timelines.get(0), partition);
    if (!firstStats.isPresent()) {
        throw new NotFoundException("partition not found");
    }
    final PartitionStatistics lastStats;
    if (timelines.size() == 1) {
        lastStats = firstStats.get();
    } else {
        lastStats = timelineService.getTopicRepository(timelines.get(timelines.size() - 1)).loadPartitionStatistics(timelines.get(timelines.size() - 1), partition).get();
    }
    return new EventTypePartitionView(eventTypeName, lastStats.getPartition(), cursorConverter.convert(firstStats.get().getFirst()).getOffset(), cursorConverter.convert(lastStats.getLast()).getOffset());
}