Examples with NakadiCursor org.zalando.nakadi.domain.NakadiCursor used on opensource projects

Example 31 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class PartitionsController method getCursorLag.

private CursorLag getCursorLag(final String eventTypeName, final String partition, final String consumedOffset) throws InternalNakadiException, NoSuchEventTypeException, InvalidCursorException, ServiceUnavailableException {
    final Cursor consumedCursor = new Cursor(partition, consumedOffset);
    final NakadiCursor consumedNakadiCursor = cursorConverter.convert(eventTypeName, consumedCursor);
    return cursorOperationsService.cursorsLag(eventTypeName, Lists.newArrayList(consumedNakadiCursor)).stream().findFirst().map(this::toCursorLag).orElseThrow(MyNakadiRuntimeException1::new);
}

Example 32 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorsServiceAT method whenCommitOldCursorsThenFalse.

@Test
public void whenCommitOldCursorsThenFalse() throws Exception {
    final NakadiCursor cursor = NakadiCursor.of(buildTimeline(etName, topic, CREATED_AT), P1, OLDEST_OFFSET);
    registerNakadiCursor(cursor);
    testCursors = ImmutableList.of(cursor);
    setPartitions(new Partition[] { new Partition(etName, P1, streamId, null, Partition.State.ASSIGNED) });
    final List<Boolean> commitResult = cursorsService.commitCursors(streamId, sid, testCursors);
    assertThat(commitResult, equalTo(ImmutableList.of(false)));
    checkCurrentOffsetInZk(P1, OLD_OFFSET);
}

Example 33 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorsServiceAT method whenFirstCursorIsNotCommittedThenNextCursorsAreNotSkipped.

@Test
public void whenFirstCursorIsNotCommittedThenNextCursorsAreNotSkipped() throws Exception {
    final NakadiCursor c1 = NakadiCursor.of(timeline, P1, OLDEST_OFFSET);
    final NakadiCursor c2 = NakadiCursor.of(timeline, P2, NEW_OFFSET);
    testCursors = ImmutableList.of(c1, c2);
    testCursors.forEach(this::registerNakadiCursor);
    setPartitions(new Partition[] { new Partition(etName, P1, streamId, null, Partition.State.ASSIGNED), new Partition(etName, P2, streamId, null, Partition.State.ASSIGNED) });
    final List<Boolean> result = cursorsService.commitCursors(streamId, sid, testCursors);
    assertFalse(result.get(0));
    assertTrue(result.get(1));
    checkCurrentOffsetInZk(P1, OLD_OFFSET);
    checkCurrentOffsetInZk(P2, NEW_OFFSET);
}

Example 34 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class MultiTimelineEventConsumer method electTopicRepositories.

private void electTopicRepositories() throws NakadiException, InvalidCursorException {
    final Map<TopicRepository, List<NakadiCursor>> newAssignment = new HashMap<>();
    borderOffsets.clear();
    // Purpose of this collection is to hold tr that definitely changed their positions and should be recreated.
    final Set<TopicPartition> actualReadPositionChanged = new HashSet<>();
    // load new topic repositories and possibly replace cursors to newer timelines.
    for (final NakadiCursor cursor : latestOffsets.values()) {
        final AtomicReference<NakadiCursor> cursorReplacement = new AtomicReference<>();
        final TopicRepository topicRepository = selectCorrectTopicRepo(cursor, cursorReplacement::set, nc -> Optional.ofNullable(nc).ifPresent(itm -> borderOffsets.put(itm.getEventTypePartition(), itm.getOffset())));
        if (!newAssignment.containsKey(topicRepository)) {
            newAssignment.put(topicRepository, new ArrayList<>());
        }
        if (cursorReplacement.get() != null) {
            actualReadPositionChanged.add(cursor.getTopicPartition());
            newAssignment.get(topicRepository).add(cursorReplacement.get());
        } else {
            newAssignment.get(topicRepository).add(cursor);
        }
    }
    final Set<TopicRepository> removedTopicRepositories = eventConsumers.keySet().stream().filter(tr -> !newAssignment.containsKey(tr)).collect(Collectors.toSet());
    // Stop and remove event consumers that are not needed anymore
    for (final TopicRepository toRemove : removedTopicRepositories) {
        stopAndRemoveConsumer(toRemove);
    }
    // Stop and remove event consumers with changed configuration
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : newAssignment.entrySet()) {
        final EventConsumer.LowLevelConsumer existingEventConsumer = eventConsumers.get(entry.getKey());
        if (null != existingEventConsumer) {
            final Set<TopicPartition> newTopicPartitions = entry.getValue().stream().map(NakadiCursor::getTopicPartition).collect(Collectors.toSet());
            final Set<TopicPartition> oldAssignment = existingEventConsumer.getAssignment();
            if (!oldAssignment.equals(newTopicPartitions) || oldAssignment.stream().anyMatch(actualReadPositionChanged::contains)) {
                stopAndRemoveConsumer(entry.getKey());
            }
        }
    }
    // Start new consumers with changed configuration.
    for (final Map.Entry<TopicRepository, List<NakadiCursor>> entry : newAssignment.entrySet()) {
        if (!eventConsumers.containsKey(entry.getKey())) {
            final TopicRepository repo = entry.getKey();
            LOG.info("Creating underlying consumer for client id {} and cursors {}", clientId, Arrays.deepToString(entry.getValue().toArray()));
            final EventConsumer.LowLevelConsumer consumer = repo.createEventConsumer(clientId, entry.getValue());
            eventConsumers.put(repo, consumer);
        }
    }
}

Example 35 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class PartitionsControllerTest method mockCursorLag.

private List<NakadiCursorLag> mockCursorLag() {
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getStorage()).thenReturn(new Storage("ccc", Storage.Type.KAFKA));
    final NakadiCursorLag lag = mock(NakadiCursorLag.class);
    final NakadiCursor firstCursor = NakadiCursor.of(timeline, "0", "0");
    final NakadiCursor lastCursor = NakadiCursor.of(timeline, "0", "1");
    when(lag.getLag()).thenReturn(42L);
    when(lag.getPartition()).thenReturn("0");
    when(lag.getFirstCursor()).thenReturn(firstCursor);
    when(lag.getLastCursor()).thenReturn(lastCursor);
    return Lists.newArrayList(lag);
}