Search in sources :

Example 16 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorConverterImplTest method testBeginConvertedVersionZero.

@Test
public void testBeginConvertedVersionZero() throws Exception {
    final String eventType = "test-et";
    final String partition = "2";
    final Storage storage = new Storage("", Storage.Type.KAFKA);
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getStorage()).thenReturn(storage);
    final EventTypeCache eventTypeCache = mock(EventTypeCache.class);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final TimelineService timelineService = mock(TimelineService.class);
    final PartitionStatistics stats = mock(PartitionStatistics.class);
    when(timelineService.getActiveTimelinesOrdered(eq(eventType))).thenReturn(Collections.singletonList(timeline));
    when(timelineService.getTopicRepository(eq(timeline))).thenReturn(topicRepository);
    when(topicRepository.loadPartitionStatistics(eq(timeline), eq(partition))).thenReturn(Optional.of(stats));
    final NakadiCursor beforeFirstCursor = NakadiCursor.of(timeline, partition, "000001");
    when(stats.getBeforeFirst()).thenReturn(beforeFirstCursor);
    final CursorConverter converter = new CursorConverterImpl(eventTypeCache, timelineService);
    final NakadiCursor nakadiCursor = converter.convert(eventType, new Cursor(partition, "BEGIN"));
    Assert.assertEquals(timeline, nakadiCursor.getTimeline());
    Assert.assertEquals(partition, nakadiCursor.getPartition());
    Assert.assertEquals("000001", nakadiCursor.getOffset());
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) EventTypeCache(org.zalando.nakadi.repository.db.EventTypeCache) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) Cursor(org.zalando.nakadi.view.Cursor) CursorConverter(org.zalando.nakadi.service.CursorConverter) Test(org.junit.Test)

Example 17 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class StreamingState method reconfigureKafkaConsumer.

private void reconfigureKafkaConsumer(final boolean forceSeek) {
    if (eventConsumer == null) {
        throw new IllegalStateException("kafkaConsumer should not be null when calling reconfigureKafkaConsumer method");
    }
    final Set<EventTypePartition> newAssignment = offsets.keySet().stream().filter(o -> !this.releasingPartitions.containsKey(o)).collect(Collectors.toSet());
    if (forceSeek) {
        // removing all the current assignments for real consumer.
        try {
            eventConsumer.reassign(Collections.emptyList());
        } catch (final NakadiException | InvalidCursorException ex) {
            throw new NakadiRuntimeException(ex);
        }
    }
    final Set<EventTypePartition> currentAssignment = eventConsumer.getAssignment();
    getLog().info("Changing kafka assignment from {} to {}", Arrays.deepToString(currentAssignment.toArray()), Arrays.deepToString(newAssignment.toArray()));
    if (!currentAssignment.equals(newAssignment)) {
        try {
            final Map<EventTypePartition, NakadiCursor> beforeFirst = getBeforeFirstCursors(newAssignment);
            final List<NakadiCursor> cursors = newAssignment.stream().map(pk -> {
                final NakadiCursor beforeFirstAvailable = beforeFirst.get(pk);
                // Checks that current cursor is still available in storage
                offsets.get(pk).ensureDataAvailable(beforeFirstAvailable);
                return offsets.get(pk).getSentOffset();
            }).collect(Collectors.toList());
            eventConsumer.reassign(cursors);
        } catch (NakadiException | InvalidCursorException ex) {
            throw new NakadiRuntimeException(ex);
        }
    }
}
Also used : ConsumedEvent(org.zalando.nakadi.domain.ConsumedEvent) Arrays(java.util.Arrays) EventTypePartition(org.zalando.nakadi.domain.EventTypePartition) IdleStreamWatcher(org.zalando.nakadi.service.subscription.IdleStreamWatcher) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) LoggerFactory(org.slf4j.LoggerFactory) Collectors.groupingBy(java.util.stream.Collectors.groupingBy) HashMap(java.util.HashMap) NakadiException(org.zalando.nakadi.exceptions.NakadiException) ZkSubscriptionClient(org.zalando.nakadi.service.subscription.zk.ZkSubscriptionClient) Partition(org.zalando.nakadi.service.subscription.model.Partition) HashSet(java.util.HashSet) Meter(com.codahale.metrics.Meter) OptionalLong(java.util.OptionalLong) JSONObject(org.json.JSONObject) SubscriptionCursorWithoutToken(org.zalando.nakadi.view.SubscriptionCursorWithoutToken) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) Map(java.util.Map) NakadiKpiPublisher(org.zalando.nakadi.service.NakadiKpiPublisher) ZkSubscription(org.zalando.nakadi.service.subscription.zk.ZkSubscription) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) StreamKpiData(org.zalando.nakadi.metrics.StreamKpiData) Set(java.util.Set) IOException(java.io.IOException) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) Collectors(java.util.stream.Collectors) TimeUnit(java.util.concurrent.TimeUnit) NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException) EventConsumer(org.zalando.nakadi.repository.EventConsumer) List(java.util.List) Stream(java.util.stream.Stream) Timeline(org.zalando.nakadi.domain.Timeline) Closeable(java.io.Closeable) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) SubscriptionCursor(org.zalando.nakadi.view.SubscriptionCursor) MetricUtils(org.zalando.nakadi.metrics.MetricUtils) Comparator(java.util.Comparator) Collections(java.util.Collections) Client(org.zalando.nakadi.security.Client) NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) EventTypePartition(org.zalando.nakadi.domain.EventTypePartition) NakadiException(org.zalando.nakadi.exceptions.NakadiException)

Example 18 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class ClosingState method reactOnOffset.

private void reactOnOffset(final EventTypePartition key) {
    if (!listeners.containsKey(key)) {
        return;
    }
    final NakadiCursor newCursor;
    try {
        newCursor = getContext().getCursorConverter().convert(key.getEventType(), listeners.get(key).getData());
    } catch (Exception ex) {
        throw new NakadiRuntimeException(ex);
    }
    if (uncommittedOffsets.containsKey(key) && getComparator().compare(uncommittedOffsets.get(key), newCursor) <= 0) {
        freePartitions(Collections.singletonList(key));
    }
    tryCompleteState();
}
Also used : NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException)

Example 19 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorOperationsService method cursorsLag.

public List<NakadiCursorLag> cursorsLag(final String eventTypeName, final List<NakadiCursor> cursors) throws InvalidCursorOperation {
    try {
        final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventTypeName);
        // Next 2 calls could be optimized to 1 storage call, instead of possible 2 calls.
        // But it is simpler not to do anything, cause timelines are not switched every day and almost all the time
        // (except retention time after switch) there will be only 1 active timeline, and this option is covered.
        final List<PartitionStatistics> oldestStats = getStatsForTimeline(timelines.get(0));
        final List<PartitionStatistics> newestStats = timelines.size() == 1 ? oldestStats : getStatsForTimeline(timelines.get(timelines.size() - 1));
        return cursors.stream().map(c -> {
            final PartitionStatistics oldestStat = oldestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            NakadiCursor newestPosition = newestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).map(PartitionEndStatistics::getLast).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            // it
            while (numberOfEventsBeforeCursor(newestPosition) == -1) {
                final int prevOrder = newestPosition.getTimeline().getOrder() - 1;
                final Timeline prevTimeline = timelines.stream().filter(t -> t.getOrder() == prevOrder).findAny().orElse(null);
                if (null == prevTimeline) {
                    break;
                }
                // We moved back, so timeline definitely have latest position set
                newestPosition = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, newestPosition.getPartition());
            }
            // calls (in case of kafka)
            return new NakadiCursorLag(oldestStat.getFirst(), newestPosition, calculateDistance(c, newestPosition));
        }).collect(Collectors.toList());
    } catch (final NakadiException e) {
        throw new MyNakadiRuntimeException1("error", e);
    }
}
Also used : Storage(org.zalando.nakadi.domain.Storage) Logger(org.slf4j.Logger) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) LoggerFactory(org.slf4j.LoggerFactory) PartitionEndStatistics(org.zalando.nakadi.domain.PartitionEndStatistics) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) Collectors(java.util.stream.Collectors) KafkaCursor(org.zalando.nakadi.repository.kafka.KafkaCursor) TIMELINE_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.TIMELINE_NOT_FOUND) List(java.util.List) UnknownStorageTypeException(org.zalando.nakadi.exceptions.runtime.UnknownStorageTypeException) Timeline(org.zalando.nakadi.domain.Timeline) PARTITION_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.PARTITION_NOT_FOUND) Service(org.springframework.stereotype.Service) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) CURSORS_WITH_DIFFERENT_PARTITION(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.CURSORS_WITH_DIFFERENT_PARTITION) Collections(java.util.Collections) Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) NakadiException(org.zalando.nakadi.exceptions.NakadiException)

Example 20 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorOperationsService method moveForward.

private NakadiCursor moveForward(final ShiftedNakadiCursor cursor) {
    NakadiCursor currentCursor = cursor.getNakadiCursor();
    long stillToAdd = cursor.getShift();
    while (currentCursor.getTimeline().getLatestPosition() != null) {
        final NakadiCursor timelineLastPosition = currentCursor.getTimeline().getLatestPosition().toNakadiCursor(currentCursor.getTimeline(), currentCursor.getPartition());
        final long distance = calculateDistance(currentCursor, timelineLastPosition);
        if (stillToAdd > distance) {
            stillToAdd -= distance;
            final Timeline nextTimeline = getTimeline(currentCursor.getEventType(), currentCursor.getTimeline().getOrder() + 1);
            currentCursor = NakadiCursor.of(nextTimeline, currentCursor.getPartition(), StaticStorageWorkerFactory.get(nextTimeline).getBeforeFirstOffset());
        } else {
            break;
        }
    }
    if (stillToAdd > 0) {
        return currentCursor.shiftWithinTimeline(stillToAdd);
    }
    return currentCursor;
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor)

Aggregations

NakadiCursor (org.zalando.nakadi.domain.NakadiCursor)56 Timeline (org.zalando.nakadi.domain.Timeline)31 Test (org.junit.Test)27 ShiftedNakadiCursor (org.zalando.nakadi.domain.ShiftedNakadiCursor)21 InvalidCursorException (org.zalando.nakadi.exceptions.InvalidCursorException)14 ServiceUnavailableException (org.zalando.nakadi.exceptions.ServiceUnavailableException)12 SubscriptionCursorWithoutToken (org.zalando.nakadi.view.SubscriptionCursorWithoutToken)12 PartitionStatistics (org.zalando.nakadi.domain.PartitionStatistics)11 List (java.util.List)10 Map (java.util.Map)10 Collectors (java.util.stream.Collectors)10 Storage (org.zalando.nakadi.domain.Storage)10 TopicRepository (org.zalando.nakadi.repository.TopicRepository)10 Optional (java.util.Optional)8 LoggerFactory (org.slf4j.LoggerFactory)8 EventTypePartition (org.zalando.nakadi.domain.EventTypePartition)8 TimelineService (org.zalando.nakadi.service.timeline.TimelineService)8 Collections (java.util.Collections)7 Logger (org.slf4j.Logger)7 InternalNakadiException (org.zalando.nakadi.exceptions.InternalNakadiException)7