Examples with Timeline org.zalando.nakadi.domain.Timeline used on opensource projects

Example 21 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class CursorOperationsService method calculateDistance.

public long calculateDistance(final NakadiCursor initialCursor, final NakadiCursor finalCursor) throws InvalidCursorOperation {
    // Validate query
    if (!initialCursor.getPartition().equals(finalCursor.getPartition())) {
        throw new InvalidCursorOperation(CURSORS_WITH_DIFFERENT_PARTITION);
    }
    long result = numberOfEventsBeforeCursor(finalCursor) - numberOfEventsBeforeCursor(initialCursor);
    final int initialOrder = initialCursor.getTimeline().getOrder();
    final int finalOrder = finalCursor.getTimeline().getOrder();
    int startOrder = Math.min(initialOrder, finalOrder);
    if (startOrder == Timeline.STARTING_ORDER) {
        startOrder += 1;
    }
    for (int order = startOrder; order < Math.max(initialOrder, finalOrder); ++order) {
        final Timeline timeline = getTimeline(initialCursor.getEventType(), order);
        final long eventsTotal = getStorageWorker(timeline).totalEventsInPartition(timeline, initialCursor.getPartition());
        result += (finalOrder > initialOrder) ? eventsTotal : -eventsTotal;
    }
    return result;
}

Example 22 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class CursorOperationsService method cursorsLag.

public List<NakadiCursorLag> cursorsLag(final String eventTypeName, final List<NakadiCursor> cursors) throws InvalidCursorOperation {
    try {
        final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventTypeName);
        // Next 2 calls could be optimized to 1 storage call, instead of possible 2 calls.
        // But it is simpler not to do anything, cause timelines are not switched every day and almost all the time
        // (except retention time after switch) there will be only 1 active timeline, and this option is covered.
        final List<PartitionStatistics> oldestStats = getStatsForTimeline(timelines.get(0));
        final List<PartitionStatistics> newestStats = timelines.size() == 1 ? oldestStats : getStatsForTimeline(timelines.get(timelines.size() - 1));
        return cursors.stream().map(c -> {
            final PartitionStatistics oldestStat = oldestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            NakadiCursor newestPosition = newestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).map(PartitionEndStatistics::getLast).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            // it
            while (numberOfEventsBeforeCursor(newestPosition) == -1) {
                final int prevOrder = newestPosition.getTimeline().getOrder() - 1;
                final Timeline prevTimeline = timelines.stream().filter(t -> t.getOrder() == prevOrder).findAny().orElse(null);
                if (null == prevTimeline) {
                    break;
                }
                // We moved back, so timeline definitely have latest position set
                newestPosition = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, newestPosition.getPartition());
            }
            // calls (in case of kafka)
            return new NakadiCursorLag(oldestStat.getFirst(), newestPosition, calculateDistance(c, newestPosition));
        }).collect(Collectors.toList());
    } catch (final NakadiException e) {
        throw new MyNakadiRuntimeException1("error", e);
    }
}

Example 23 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class CursorOperationsService method moveForward.

private NakadiCursor moveForward(final ShiftedNakadiCursor cursor) {
    NakadiCursor currentCursor = cursor.getNakadiCursor();
    long stillToAdd = cursor.getShift();
    while (currentCursor.getTimeline().getLatestPosition() != null) {
        final NakadiCursor timelineLastPosition = currentCursor.getTimeline().getLatestPosition().toNakadiCursor(currentCursor.getTimeline(), currentCursor.getPartition());
        final long distance = calculateDistance(currentCursor, timelineLastPosition);
        if (stillToAdd > distance) {
            stillToAdd -= distance;
            final Timeline nextTimeline = getTimeline(currentCursor.getEventType(), currentCursor.getTimeline().getOrder() + 1);
            currentCursor = NakadiCursor.of(nextTimeline, currentCursor.getPartition(), StaticStorageWorkerFactory.get(nextTimeline).getBeforeFirstOffset());
        } else {
            break;
        }
    }
    if (stillToAdd > 0) {
        return currentCursor.shiftWithinTimeline(stillToAdd);
    }
    return currentCursor;
}

Example 24 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class EventTypeService method updateTimelinesCleanup.

private void updateTimelinesCleanup(final String eventType, final Long newRetentionTime, final Long oldRetentionTime) throws InternalNakadiException, NoSuchEventTypeException {
    if (newRetentionTime != null && !newRetentionTime.equals(oldRetentionTime)) {
        final long retentionDiffMs = newRetentionTime - oldRetentionTime;
        final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventType);
        for (final Timeline timeline : timelines) {
            if (timeline.getCleanedUpAt() != null) {
                timeline.setCleanedUpAt(new Date(timeline.getCleanedUpAt().getTime() + retentionDiffMs));
                timelineService.updateTimeline(timeline);
            }
        }
    }
}

Example 25 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class SubscriptionService method loadPartitionEndStatistics.

private List<PartitionEndStatistics> loadPartitionEndStatistics(final Collection<EventType> eventTypes) throws ServiceUnavailableException {
    final List<PartitionEndStatistics> topicPartitions = new ArrayList<>();
    final Map<TopicRepository, List<Timeline>> timelinesByRepo = eventTypes.stream().map(timelineService::getActiveTimeline).collect(Collectors.groupingBy(timelineService::getTopicRepository));
    for (final Map.Entry<TopicRepository, List<Timeline>> repoEntry : timelinesByRepo.entrySet()) {
        final TopicRepository topicRepository = repoEntry.getKey();
        final List<Timeline> timelinesForRepo = repoEntry.getValue();
        topicPartitions.addAll(topicRepository.loadTopicEndStatistics(timelinesForRepo));
    }
    return topicPartitions;
}