Examples with NakadiCursor org.zalando.nakadi.domain.NakadiCursor used on opensource projects

Example 21 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorsService method resetCursors.

public void resetCursors(final String subscriptionId, final List<NakadiCursor> cursors) throws ServiceUnavailableException, NoSuchSubscriptionException, UnableProcessException, OperationTimeoutException, ZookeeperException, InternalNakadiException, NoSuchEventTypeException, InvalidCursorException {
    final Subscription subscription = subscriptionRepository.getSubscription(subscriptionId);
    validateCursorsBelongToSubscription(subscription, cursors);
    for (final NakadiCursor cursor : cursors) {
        cursor.checkStorageAvailability();
    }
    final Map<TopicRepository, List<NakadiCursor>> topicRepositories = cursors.stream().collect(Collectors.groupingBy(c -> timelineService.getTopicRepository(c.getTimeline())));
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : topicRepositories.entrySet()) {
        entry.getKey().validateReadCursors(entry.getValue());
    }
    final ZkSubscriptionClient zkClient = zkSubscriptionFactory.createClient(subscription, "subscription." + subscriptionId + ".reset_cursors");
    // In case if subscription was never initialized - initialize it
    zkClient.runLocked(() -> StartingState.initializeSubscriptionLocked(zkClient, subscription, timelineService, cursorConverter));
    // add 1 second to commit timeout in order to give time to finish reset if there is uncommitted events
    if (!cursors.isEmpty()) {
        final long timeout = TimeUnit.SECONDS.toMillis(nakadiSettings.getDefaultCommitTimeoutSeconds()) + TimeUnit.SECONDS.toMillis(1);
        zkClient.resetCursors(cursors.stream().map(cursorConverter::convertToNoToken).collect(Collectors.toList()), timeout);
    }
}

Example 22 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class TopicRepositoryHolder method createStoragePosition.

public Timeline.StoragePosition createStoragePosition(final Timeline timeline) {
    try {
        final Storage storage = timeline.getStorage();
        final List<NakadiCursor> offsets = getTopicRepository(storage).loadTopicStatistics(Collections.singleton(timeline)).stream().map(PartitionStatistics::getLast).collect(Collectors.toList());
        return getTopicRepositoryCreator(storage.getType()).createStoragePosition(offsets);
    } catch (final ServiceUnavailableException e) {
        throw new NakadiRuntimeException(e);
    }
}

Example 23 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class VersionZeroConverter method convertBatched.

public List<NakadiCursor> convertBatched(final List<SubscriptionCursorWithoutToken> cursors) throws InvalidCursorException, InternalNakadiException, NoSuchEventTypeException, ServiceUnavailableException {
    final NakadiCursor[] result = new NakadiCursor[cursors.size()];
    for (int idx = 0; idx < cursors.size(); ++idx) {
        final SubscriptionCursorWithoutToken cursor = cursors.get(idx);
        if (Cursor.BEFORE_OLDEST_OFFSET.equalsIgnoreCase(cursor.getOffset())) {
            // Preform begin checks afterwards to optimize calls
            continue;
        }
        if (!NUMBERS_ONLY_PATTERN.matcher(cursor.getOffset()).matches()) {
            throw new InvalidCursorException(CursorError.INVALID_OFFSET, cursor);
        }
    }
    // now it is time for massive convert.
    final LinkedHashMap<SubscriptionCursorWithoutToken, NakadiCursor> beginsToConvert = new LinkedHashMap<>();
    final Map<SubscriptionCursorWithoutToken, Timeline> cursorTimelines = new HashMap<>();
    final Map<TopicRepository, List<SubscriptionCursorWithoutToken>> repos = new HashMap<>();
    for (int i = 0; i < result.length; ++i) {
        if (null == result[i]) {
            // cursor requires database hit
            final SubscriptionCursorWithoutToken cursor = cursors.get(i);
            final Timeline timeline = timelineService.getActiveTimelinesOrdered(cursor.getEventType()).get(0);
            final TopicRepository topicRepo = timelineService.getTopicRepository(timeline);
            beginsToConvert.put(cursor, null);
            cursorTimelines.put(cursor, timeline);
            repos.computeIfAbsent(topicRepo, k -> new ArrayList<>()).add(cursor);
        }
    }
    for (final Map.Entry<TopicRepository, List<SubscriptionCursorWithoutToken>> entry : repos.entrySet()) {
        final List<Optional<PartitionStatistics>> stats = entry.getKey().loadPartitionStatistics(entry.getValue().stream().map(scwt -> new TopicRepository.TimelinePartition(cursorTimelines.get(scwt), scwt.getPartition())).collect(Collectors.toList()));
        for (int idx = 0; idx < entry.getValue().size(); ++idx) {
            // Reinsert doesn't change the order
            beginsToConvert.put(entry.getValue().get(idx), stats.get(idx).orElseThrow(() -> new InvalidCursorException(PARTITION_NOT_FOUND)).getBeforeFirst());
        }
    }
    final Iterator<NakadiCursor> missingBegins = beginsToConvert.values().iterator();
    return Stream.of(result).map(it -> null == it ? missingBegins.next() : it).collect(Collectors.toList());
}

Example 24 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class VersionOneConverter method findCorrectTimelinedCursor.

private NakadiCursor findCorrectTimelinedCursor(final String eventType, final int order, final String partition, final String offset) throws InternalNakadiException, NoSuchEventTypeException, InvalidCursorException {
    final List<Timeline> timelines = eventTypeCache.getTimelinesOrdered(eventType);
    final Iterator<Timeline> timelineIterator = timelines.iterator();
    Timeline timeline = null;
    while (timelineIterator.hasNext()) {
        final Timeline t = timelineIterator.next();
        if (t.getOrder() == order) {
            timeline = t;
            break;
        }
    }
    if (null == timeline) {
        throw new InvalidCursorException(CursorError.UNAVAILABLE);
    }
    NakadiCursor cursor = NakadiCursor.of(timeline, partition, offset);
    while (cursor.isLast()) {
        // Will not check this call, because latest offset is not set for last timeline
        timeline = timelineIterator.next();
        cursor = NakadiCursor.of(timeline, partition, StaticStorageWorkerFactory.get(timeline).getBeforeFirstOffset());
    }
    return cursor;
}

Example 25 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class NakadiCursorComparator method compareOrdered.

private int compareOrdered(final NakadiCursor c1, final NakadiCursor c2) {
    // If c2 moved from -1, than it is definitely greater.
    if (!c2.isInitial()) {
        return -1;
    }
    Iterator<Timeline> timelineIterator = null;
    NakadiCursor first = c1;
    // Handle obsolete timeline information
    if (first.getTimeline().getLatestPosition() == null) {
        timelineIterator = createTimelinesIterator(first.getEventType(), first.getTimeline().getOrder());
        first = NakadiCursor.of(timelineIterator.next(), first.getPartition(), first.getOffset());
    }
    while (first.getTimeline().getOrder() != c2.getTimeline().getOrder()) {
        if (!first.isLast()) {
            return -1;
        }
        if (null == timelineIterator) {
            timelineIterator = createTimelinesIterator(first.getEventType(), first.getTimeline().getOrder() + 1);
        }
        final Timeline nextTimeline = timelineIterator.next();
        final String initialOffset = StaticStorageWorkerFactory.get(nextTimeline).getBeforeFirstOffset();
        first = NakadiCursor.of(nextTimeline, first.getPartition(), initialOffset);
    }
    return first.getOffset().compareTo(c2.getOffset());
}