Examples with TopicRepository org.zalando.nakadi.repository.TopicRepository used on opensource projects

Example 11 with TopicRepository

use of org.zalando.nakadi.repository.TopicRepository in project nakadi by zalando.

the class CursorsService method resetCursors.

public void resetCursors(final String subscriptionId, final List<NakadiCursor> cursors) throws ServiceUnavailableException, NoSuchSubscriptionException, UnableProcessException, OperationTimeoutException, ZookeeperException, InternalNakadiException, NoSuchEventTypeException, InvalidCursorException {
    final Subscription subscription = subscriptionRepository.getSubscription(subscriptionId);
    validateCursorsBelongToSubscription(subscription, cursors);
    for (final NakadiCursor cursor : cursors) {
        cursor.checkStorageAvailability();
    }
    final Map<TopicRepository, List<NakadiCursor>> topicRepositories = cursors.stream().collect(Collectors.groupingBy(c -> timelineService.getTopicRepository(c.getTimeline())));
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : topicRepositories.entrySet()) {
        entry.getKey().validateReadCursors(entry.getValue());
    }
    final ZkSubscriptionClient zkClient = zkSubscriptionFactory.createClient(subscription, "subscription." + subscriptionId + ".reset_cursors");
    // In case if subscription was never initialized - initialize it
    zkClient.runLocked(() -> StartingState.initializeSubscriptionLocked(zkClient, subscription, timelineService, cursorConverter));
    // add 1 second to commit timeout in order to give time to finish reset if there is uncommitted events
    if (!cursors.isEmpty()) {
        final long timeout = TimeUnit.SECONDS.toMillis(nakadiSettings.getDefaultCommitTimeoutSeconds()) + TimeUnit.SECONDS.toMillis(1);
        zkClient.resetCursors(cursors.stream().map(cursorConverter::convertToNoToken).collect(Collectors.toList()), timeout);
    }
}

Example 12 with TopicRepository

use of org.zalando.nakadi.repository.TopicRepository in project nakadi by zalando.

the class SubscriptionService method loadPartitionEndStatistics.

private List<PartitionEndStatistics> loadPartitionEndStatistics(final Collection<EventType> eventTypes) throws ServiceUnavailableException {
    final List<PartitionEndStatistics> topicPartitions = new ArrayList<>();
    final Map<TopicRepository, List<Timeline>> timelinesByRepo = eventTypes.stream().map(timelineService::getActiveTimeline).collect(Collectors.groupingBy(timelineService::getTopicRepository));
    for (final Map.Entry<TopicRepository, List<Timeline>> repoEntry : timelinesByRepo.entrySet()) {
        final TopicRepository topicRepository = repoEntry.getKey();
        final List<Timeline> timelinesForRepo = repoEntry.getValue();
        topicPartitions.addAll(topicRepository.loadTopicEndStatistics(timelinesForRepo));
    }
    return topicPartitions;
}

Example 13 with TopicRepository

use of org.zalando.nakadi.repository.TopicRepository in project nakadi by zalando.

the class VersionZeroConverter method convertBatched.

public List<NakadiCursor> convertBatched(final List<SubscriptionCursorWithoutToken> cursors) throws InvalidCursorException, InternalNakadiException, NoSuchEventTypeException, ServiceUnavailableException {
    final NakadiCursor[] result = new NakadiCursor[cursors.size()];
    for (int idx = 0; idx < cursors.size(); ++idx) {
        final SubscriptionCursorWithoutToken cursor = cursors.get(idx);
        if (Cursor.BEFORE_OLDEST_OFFSET.equalsIgnoreCase(cursor.getOffset())) {
            // Preform begin checks afterwards to optimize calls
            continue;
        }
        if (!NUMBERS_ONLY_PATTERN.matcher(cursor.getOffset()).matches()) {
            throw new InvalidCursorException(CursorError.INVALID_OFFSET, cursor);
        }
    }
    // now it is time for massive convert.
    final LinkedHashMap<SubscriptionCursorWithoutToken, NakadiCursor> beginsToConvert = new LinkedHashMap<>();
    final Map<SubscriptionCursorWithoutToken, Timeline> cursorTimelines = new HashMap<>();
    final Map<TopicRepository, List<SubscriptionCursorWithoutToken>> repos = new HashMap<>();
    for (int i = 0; i < result.length; ++i) {
        if (null == result[i]) {
            // cursor requires database hit
            final SubscriptionCursorWithoutToken cursor = cursors.get(i);
            final Timeline timeline = timelineService.getActiveTimelinesOrdered(cursor.getEventType()).get(0);
            final TopicRepository topicRepo = timelineService.getTopicRepository(timeline);
            beginsToConvert.put(cursor, null);
            cursorTimelines.put(cursor, timeline);
            repos.computeIfAbsent(topicRepo, k -> new ArrayList<>()).add(cursor);
        }
    }
    for (final Map.Entry<TopicRepository, List<SubscriptionCursorWithoutToken>> entry : repos.entrySet()) {
        final List<Optional<PartitionStatistics>> stats = entry.getKey().loadPartitionStatistics(entry.getValue().stream().map(scwt -> new TopicRepository.TimelinePartition(cursorTimelines.get(scwt), scwt.getPartition())).collect(Collectors.toList()));
        for (int idx = 0; idx < entry.getValue().size(); ++idx) {
            // Reinsert doesn't change the order
            beginsToConvert.put(entry.getValue().get(idx), stats.get(idx).orElseThrow(() -> new InvalidCursorException(PARTITION_NOT_FOUND)).getBeforeFirst());
        }
    }
    final Iterator<NakadiCursor> missingBegins = beginsToConvert.values().iterator();
    return Stream.of(result).map(it -> null == it ? missingBegins.next() : it).collect(Collectors.toList());
}

Example 14 with TopicRepository

use of org.zalando.nakadi.repository.TopicRepository in project nakadi by zalando.

the class CursorsServiceAT method before.

@Before
public void before() throws Exception {
    sid = randomUUID();
    streamId = randomUUID();
    etName = randomValidEventTypeName();
    topic = randomUUID();
    cursorConverter = mock(CursorConverter.class);
    testCursors = ImmutableList.of(NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P1, NEW_OFFSET));
    final EventType eventType = mock(EventType.class);
    when(eventType.getName()).thenReturn(etName);
    final ZooKeeperHolder zkHolder = mock(ZooKeeperHolder.class);
    when(zkHolder.get()).thenReturn(CURATOR);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final TimelineService timelineService = mock(TimelineService.class);
    when(timelineService.getTopicRepository((Timeline) any())).thenReturn(topicRepository);
    timeline = buildTimeline(etName, topic, CREATED_AT);
    when(timelineService.getActiveTimeline(any(EventType.class))).thenReturn(timeline);
    final Subscription subscription = mock(Subscription.class);
    when(subscription.getId()).thenReturn(sid);
    when(subscription.getEventTypes()).thenReturn(ImmutableSet.of(etName));
    final SubscriptionDbRepository subscriptionRepo = mock(SubscriptionDbRepository.class);
    when(subscriptionRepo.getSubscription(sid)).thenReturn(subscription);
    final SubscriptionClientFactory zkSubscriptionFactory = new SubscriptionClientFactory(zkHolder, MAPPER);
    uuidGenerator = mock(UUIDGenerator.class);
    when(uuidGenerator.isUUID(any())).thenReturn(true);
    cursorsService = new CursorsService(subscriptionRepo, null, mock(NakadiSettings.class), zkSubscriptionFactory, cursorConverter, uuidGenerator, null);
    // Register cursors in converter
    registerNakadiCursor(NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P1, NEW_OFFSET));
    registerNakadiCursor(NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P1, OLD_OFFSET));
    registerNakadiCursor(NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P2, NEW_OFFSET));
    registerNakadiCursor(NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P2, OLD_OFFSET));
    // bootstrap data in ZK
    CURATOR.create().creatingParentsIfNeeded().forPath(offsetPath(P1), OLD_OFFSET.getBytes(UTF_8));
    CURATOR.create().creatingParentsIfNeeded().forPath(offsetPath(P2), OLD_OFFSET.getBytes(UTF_8));
    CURATOR.create().creatingParentsIfNeeded().forPath(sessionPath(streamId));
}

Example 15 with TopicRepository

use of org.zalando.nakadi.repository.TopicRepository in project nakadi by zalando.

the class EventStreamController method getStreamingStart.

@VisibleForTesting
List<NakadiCursor> getStreamingStart(final EventType eventType, final String cursorsStr) throws UnparseableCursorException, ServiceUnavailableException, InvalidCursorException, InternalNakadiException, NoSuchEventTypeException {
    List<Cursor> cursors = null;
    if (cursorsStr != null) {
        try {
            cursors = jsonMapper.readValue(cursorsStr, new TypeReference<ArrayList<Cursor>>() {
            });
        } catch (final IOException ex) {
            throw new UnparseableCursorException("incorrect syntax of X-nakadi-cursors header", ex, cursorsStr);
        }
        // Unfortunately, In order to have consistent exception checking, one can not just call validator
        for (final Cursor cursor : cursors) {
            if (null == cursor.getPartition()) {
                throw new InvalidCursorException(CursorError.NULL_PARTITION, cursor);
            } else if (null == cursor.getOffset()) {
                throw new InvalidCursorException(CursorError.NULL_OFFSET, cursor);
            }
        }
    }
    final Timeline latestTimeline = timelineService.getActiveTimeline(eventType);
    if (null != cursors) {
        if (cursors.isEmpty()) {
            throw new InvalidCursorException(CursorError.INVALID_FORMAT);
        }
        final List<NakadiCursor> result = new ArrayList<>();
        for (final Cursor c : cursors) {
            result.add(cursorConverter.convert(eventType.getName(), c));
        }
        for (final NakadiCursor c : result) {
            if (c.getTimeline().isDeleted()) {
                throw new InvalidCursorException(CursorError.UNAVAILABLE, c);
            }
        }
        final Map<Storage, List<NakadiCursor>> groupedCursors = result.stream().collect(Collectors.groupingBy(c -> c.getTimeline().getStorage()));
        for (final Map.Entry<Storage, List<NakadiCursor>> entry : groupedCursors.entrySet()) {
            timelineService.getTopicRepository(entry.getKey()).validateReadCursors(entry.getValue());
        }
        return result;
    } else {
        final TopicRepository latestTopicRepository = timelineService.getTopicRepository(latestTimeline);
        // if no cursors provided - read from the newest available events
        return latestTopicRepository.loadTopicStatistics(Collections.singletonList(latestTimeline)).stream().map(PartitionStatistics::getLast).collect(Collectors.toList());
    }
}