Search in sources :

Example 1 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class StreamingStateTest method ensureOffsetsSubscriptionsAreRefreshedAndClosed.

@Test
public void ensureOffsetsSubscriptionsAreRefreshedAndClosed() throws InternalNakadiException, NoSuchEventTypeException, ServiceUnavailableException, InvalidCursorException {
    final ZkSubscription<SubscriptionCursorWithoutToken> offsetSubscription = mock(ZkSubscription.class);
    final EventTypePartition pk = new EventTypePartition("t", "0");
    Mockito.when(zkMock.subscribeForOffsetChanges(Mockito.eq(pk), Mockito.any())).thenReturn(offsetSubscription);
    final EventConsumer.ReassignableEventConsumer consumer = mock(EventConsumer.ReassignableEventConsumer.class);
    when(consumer.getAssignment()).thenReturn(Collections.emptySet());
    when(timelineService.createEventConsumer(any())).thenReturn(consumer);
    when(subscription.getEventTypes()).thenReturn(Collections.singleton("t"));
    final Storage storage = mock(Storage.class);
    when(storage.getType()).thenReturn(Storage.Type.KAFKA);
    final Timeline timeline = new Timeline("t", 0, storage, "t", new Date());
    when(timelineService.getActiveTimelinesOrdered(eq("t"))).thenReturn(Collections.singletonList(timeline));
    final TopicRepository topicRepository = mock(TopicRepository.class);
    when(timelineService.getTopicRepository(eq(timeline))).thenReturn(topicRepository);
    final PartitionStatistics stats = mock(PartitionStatistics.class);
    final NakadiCursor beforeFirstCursor = NakadiCursor.of(timeline, "0", "0");
    when(stats.getBeforeFirst()).thenReturn(beforeFirstCursor);
    when(topicRepository.loadTopicStatistics(any())).thenReturn(Lists.newArrayList(stats));
    state.onEnter();
    final NakadiCursor anyCursor = NakadiCursor.of(timeline, "0", "0");
    when(cursorConverter.convert(any(SubscriptionCursorWithoutToken.class))).thenReturn(anyCursor);
    state.refreshTopologyUnlocked(new Partition[] { new Partition(pk.getEventType(), pk.getPartition(), SESSION_ID, null, Partition.State.ASSIGNED) });
    Mockito.verify(zkMock, Mockito.times(1)).subscribeForOffsetChanges(Mockito.eq(pk), Mockito.any());
    Mockito.verify(offsetSubscription, Mockito.times(0)).close();
    Mockito.verify(offsetSubscription, Mockito.times(0)).getData();
    state.offsetChanged(pk);
    Mockito.verify(zkMock, Mockito.times(1)).subscribeForOffsetChanges(Mockito.eq(pk), Mockito.any());
    Mockito.verify(offsetSubscription, Mockito.times(0)).close();
    Mockito.verify(offsetSubscription, Mockito.times(1)).getData();
    // Verify that offset change listener is removed
    state.refreshTopologyUnlocked(new Partition[0]);
    Mockito.verify(zkMock, Mockito.times(1)).subscribeForOffsetChanges(Mockito.eq(pk), Mockito.any());
    Mockito.verify(offsetSubscription, Mockito.times(1)).close();
    Mockito.verify(offsetSubscription, Mockito.times(1)).getData();
}
Also used : SubscriptionCursorWithoutToken(org.zalando.nakadi.view.SubscriptionCursorWithoutToken) EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventTypePartition(org.zalando.nakadi.domain.EventTypePartition) Partition(org.zalando.nakadi.service.subscription.model.Partition) Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TopicRepository(org.zalando.nakadi.repository.TopicRepository) EventTypePartition(org.zalando.nakadi.domain.EventTypePartition) Date(java.util.Date) Test(org.junit.Test)

Example 2 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class StartingStateTest method testGetSubscriptionOffsetsBegin.

@Test
public void testGetSubscriptionOffsetsBegin() throws Exception {
    when(subscription.getReadFrom()).thenReturn(SubscriptionBase.InitialPosition.BEGIN);
    final NakadiCursor beforeBegin0 = mock(NakadiCursor.class);
    final SubscriptionCursorWithoutToken beforeBegin0Converted = mock(SubscriptionCursorWithoutToken.class);
    when(cursorConverter.convertToNoToken(eq(beforeBegin0))).thenReturn(beforeBegin0Converted);
    final NakadiCursor beforeBegin1 = mock(NakadiCursor.class);
    final SubscriptionCursorWithoutToken beforeBegin1Converted = mock(SubscriptionCursorWithoutToken.class);
    when(cursorConverter.convertToNoToken(eq(beforeBegin1))).thenReturn(beforeBegin1Converted);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final PartitionStatistics resultForTopic0 = mock(PartitionStatistics.class);
    when(resultForTopic0.getBeforeFirst()).thenReturn(beforeBegin0);
    final PartitionStatistics resultForTopic1 = mock(PartitionStatistics.class);
    when(resultForTopic1.getBeforeFirst()).thenReturn(beforeBegin1);
    when(topicRepository.loadTopicStatistics(any())).thenReturn(Lists.newArrayList(resultForTopic0, resultForTopic1));
    when(timelineService.getTopicRepository(eq(timelineEt00))).thenReturn(topicRepository);
    final Storage storage = mock(Storage.class);
    when(timelineEt00.getStorage()).thenReturn(storage);
    when(timelineEt10.getStorage()).thenReturn(storage);
    final List<SubscriptionCursorWithoutToken> cursors = StartingState.calculateStartPosition(subscription, timelineService, cursorConverter);
    Assert.assertEquals(cursors.size(), 2);
    Assert.assertEquals(beforeBegin0Converted, cursors.get(0));
    Assert.assertEquals(beforeBegin1Converted, cursors.get(1));
}
Also used : SubscriptionCursorWithoutToken(org.zalando.nakadi.view.SubscriptionCursorWithoutToken) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) Storage(org.zalando.nakadi.domain.Storage) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TopicRepository(org.zalando.nakadi.repository.TopicRepository) Test(org.junit.Test)

Example 3 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class StartingStateTest method testGetSubscriptionOffsetsEnd.

@Test
public void testGetSubscriptionOffsetsEnd() throws Exception {
    when(subscription.getReadFrom()).thenReturn(SubscriptionBase.InitialPosition.END);
    final NakadiCursor end0 = mock(NakadiCursor.class);
    final SubscriptionCursorWithoutToken end0Converted = mock(SubscriptionCursorWithoutToken.class);
    when(cursorConverter.convertToNoToken(eq(end0))).thenReturn(end0Converted);
    final NakadiCursor end1 = mock(NakadiCursor.class);
    final SubscriptionCursorWithoutToken end1Converted = mock(SubscriptionCursorWithoutToken.class);
    when(cursorConverter.convertToNoToken(eq(end1))).thenReturn(end1Converted);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final PartitionStatistics statsForEt0 = mock(PartitionStatistics.class);
    when(statsForEt0.getLast()).thenReturn(end0);
    final PartitionStatistics statsForTopic1 = mock(PartitionStatistics.class);
    when(statsForTopic1.getLast()).thenReturn(end1);
    when(topicRepository.loadTopicEndStatistics(any())).thenReturn(Lists.newArrayList(statsForEt0, statsForTopic1));
    when(timelineService.getTopicRepository(eq(timelineEt01))).thenReturn(topicRepository);
    final Storage storage = mock(Storage.class);
    when(timelineEt01.getStorage()).thenReturn(storage);
    when(timelineEt11.getStorage()).thenReturn(storage);
    final List<SubscriptionCursorWithoutToken> cursors = StartingState.calculateStartPosition(subscription, timelineService, cursorConverter);
    Assert.assertEquals(cursors.size(), 2);
    Assert.assertEquals(end0Converted, cursors.get(0));
    Assert.assertEquals(end1Converted, cursors.get(1));
}
Also used : SubscriptionCursorWithoutToken(org.zalando.nakadi.view.SubscriptionCursorWithoutToken) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) Storage(org.zalando.nakadi.domain.Storage) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TopicRepository(org.zalando.nakadi.repository.TopicRepository) Test(org.junit.Test)

Example 4 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class CursorConverterImplTest method testBeginConvertedVersionZero.

@Test
public void testBeginConvertedVersionZero() throws Exception {
    final String eventType = "test-et";
    final String partition = "2";
    final Storage storage = new Storage("", Storage.Type.KAFKA);
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getStorage()).thenReturn(storage);
    final EventTypeCache eventTypeCache = mock(EventTypeCache.class);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final TimelineService timelineService = mock(TimelineService.class);
    final PartitionStatistics stats = mock(PartitionStatistics.class);
    when(timelineService.getActiveTimelinesOrdered(eq(eventType))).thenReturn(Collections.singletonList(timeline));
    when(timelineService.getTopicRepository(eq(timeline))).thenReturn(topicRepository);
    when(topicRepository.loadPartitionStatistics(eq(timeline), eq(partition))).thenReturn(Optional.of(stats));
    final NakadiCursor beforeFirstCursor = NakadiCursor.of(timeline, partition, "000001");
    when(stats.getBeforeFirst()).thenReturn(beforeFirstCursor);
    final CursorConverter converter = new CursorConverterImpl(eventTypeCache, timelineService);
    final NakadiCursor nakadiCursor = converter.convert(eventType, new Cursor(partition, "BEGIN"));
    Assert.assertEquals(timeline, nakadiCursor.getTimeline());
    Assert.assertEquals(partition, nakadiCursor.getPartition());
    Assert.assertEquals("000001", nakadiCursor.getOffset());
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) EventTypeCache(org.zalando.nakadi.repository.db.EventTypeCache) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) Cursor(org.zalando.nakadi.view.Cursor) CursorConverter(org.zalando.nakadi.service.CursorConverter) Test(org.junit.Test)

Example 5 with PartitionStatistics

use of org.zalando.nakadi.domain.PartitionStatistics in project nakadi by zalando.

the class CursorOperationsService method cursorsLag.

public List<NakadiCursorLag> cursorsLag(final String eventTypeName, final List<NakadiCursor> cursors) throws InvalidCursorOperation {
    try {
        final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventTypeName);
        // Next 2 calls could be optimized to 1 storage call, instead of possible 2 calls.
        // But it is simpler not to do anything, cause timelines are not switched every day and almost all the time
        // (except retention time after switch) there will be only 1 active timeline, and this option is covered.
        final List<PartitionStatistics> oldestStats = getStatsForTimeline(timelines.get(0));
        final List<PartitionStatistics> newestStats = timelines.size() == 1 ? oldestStats : getStatsForTimeline(timelines.get(timelines.size() - 1));
        return cursors.stream().map(c -> {
            final PartitionStatistics oldestStat = oldestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            NakadiCursor newestPosition = newestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).map(PartitionEndStatistics::getLast).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            // it
            while (numberOfEventsBeforeCursor(newestPosition) == -1) {
                final int prevOrder = newestPosition.getTimeline().getOrder() - 1;
                final Timeline prevTimeline = timelines.stream().filter(t -> t.getOrder() == prevOrder).findAny().orElse(null);
                if (null == prevTimeline) {
                    break;
                }
                // We moved back, so timeline definitely have latest position set
                newestPosition = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, newestPosition.getPartition());
            }
            // calls (in case of kafka)
            return new NakadiCursorLag(oldestStat.getFirst(), newestPosition, calculateDistance(c, newestPosition));
        }).collect(Collectors.toList());
    } catch (final NakadiException e) {
        throw new MyNakadiRuntimeException1("error", e);
    }
}
Also used : Storage(org.zalando.nakadi.domain.Storage) Logger(org.slf4j.Logger) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) LoggerFactory(org.slf4j.LoggerFactory) PartitionEndStatistics(org.zalando.nakadi.domain.PartitionEndStatistics) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) Collectors(java.util.stream.Collectors) KafkaCursor(org.zalando.nakadi.repository.kafka.KafkaCursor) TIMELINE_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.TIMELINE_NOT_FOUND) List(java.util.List) UnknownStorageTypeException(org.zalando.nakadi.exceptions.runtime.UnknownStorageTypeException) Timeline(org.zalando.nakadi.domain.Timeline) PARTITION_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.PARTITION_NOT_FOUND) Service(org.springframework.stereotype.Service) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) CURSORS_WITH_DIFFERENT_PARTITION(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.CURSORS_WITH_DIFFERENT_PARTITION) Collections(java.util.Collections) Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) NakadiException(org.zalando.nakadi.exceptions.NakadiException)

Aggregations

PartitionStatistics (org.zalando.nakadi.domain.PartitionStatistics)15 NakadiCursor (org.zalando.nakadi.domain.NakadiCursor)12 Timeline (org.zalando.nakadi.domain.Timeline)12 TopicRepository (org.zalando.nakadi.repository.TopicRepository)10 List (java.util.List)8 Collectors (java.util.stream.Collectors)8 ServiceUnavailableException (org.zalando.nakadi.exceptions.ServiceUnavailableException)8 Collections (java.util.Collections)7 InvalidCursorException (org.zalando.nakadi.exceptions.InvalidCursorException)7 ArrayList (java.util.ArrayList)6 Test (org.junit.Test)6 Storage (org.zalando.nakadi.domain.Storage)6 Map (java.util.Map)5 Optional (java.util.Optional)5 Logger (org.slf4j.Logger)5 LoggerFactory (org.slf4j.LoggerFactory)5 PartitionEndStatistics (org.zalando.nakadi.domain.PartitionEndStatistics)5 TimelineService (org.zalando.nakadi.service.timeline.TimelineService)5 HashMap (java.util.HashMap)4 Set (java.util.Set)4