Search in sources :

Example 6 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class CursorOperationsServiceTest method mockTimeline.

private Timeline mockTimeline(final int order, @Nullable final Long latestOffset) {
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getOrder()).thenReturn(order);
    final Storage storage = new Storage();
    storage.setType(Storage.Type.KAFKA);
    when(timeline.getStorage()).thenReturn(storage);
    if (latestOffset == null) {
        when(timeline.isActive()).thenReturn(false);
        when(timeline.getLatestPosition()).thenReturn(null);
    } else {
        when(timeline.isActive()).thenReturn(true);
        when(timeline.getLatestPosition()).thenReturn(new Timeline.KafkaStoragePosition(Collections.singletonList(latestOffset)));
    }
    when(timeline.isActive()).thenReturn(null == latestOffset);
    final TopicRepository repository = new KafkaTopicRepository(mock(ZooKeeperHolder.class), mock(KafkaFactory.class), mock(NakadiSettings.class), mock(KafkaSettings.class), mock(ZookeeperSettings.class), mock(UUIDGenerator.class));
    when(timelineService.getTopicRepository(timeline)).thenReturn(repository);
    return timeline;
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) KafkaFactory(org.zalando.nakadi.repository.kafka.KafkaFactory) Storage(org.zalando.nakadi.domain.Storage) KafkaTopicRepository(org.zalando.nakadi.repository.kafka.KafkaTopicRepository) KafkaSettings(org.zalando.nakadi.repository.kafka.KafkaSettings) UUIDGenerator(org.zalando.nakadi.util.UUIDGenerator) TopicRepository(org.zalando.nakadi.repository.TopicRepository) KafkaTopicRepository(org.zalando.nakadi.repository.kafka.KafkaTopicRepository) ZookeeperSettings(org.zalando.nakadi.repository.zookeeper.ZookeeperSettings) ZooKeeperHolder(org.zalando.nakadi.repository.zookeeper.ZooKeeperHolder) NakadiSettings(org.zalando.nakadi.config.NakadiSettings)

Example 7 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class CursorConverterImplTest method testBeginConvertedVersionZero.

@Test
public void testBeginConvertedVersionZero() throws Exception {
    final String eventType = "test-et";
    final String partition = "2";
    final Storage storage = new Storage("", Storage.Type.KAFKA);
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getStorage()).thenReturn(storage);
    final EventTypeCache eventTypeCache = mock(EventTypeCache.class);
    final TopicRepository topicRepository = mock(TopicRepository.class);
    final TimelineService timelineService = mock(TimelineService.class);
    final PartitionStatistics stats = mock(PartitionStatistics.class);
    when(timelineService.getActiveTimelinesOrdered(eq(eventType))).thenReturn(Collections.singletonList(timeline));
    when(timelineService.getTopicRepository(eq(timeline))).thenReturn(topicRepository);
    when(topicRepository.loadPartitionStatistics(eq(timeline), eq(partition))).thenReturn(Optional.of(stats));
    final NakadiCursor beforeFirstCursor = NakadiCursor.of(timeline, partition, "000001");
    when(stats.getBeforeFirst()).thenReturn(beforeFirstCursor);
    final CursorConverter converter = new CursorConverterImpl(eventTypeCache, timelineService);
    final NakadiCursor nakadiCursor = converter.convert(eventType, new Cursor(partition, "BEGIN"));
    Assert.assertEquals(timeline, nakadiCursor.getTimeline());
    Assert.assertEquals(partition, nakadiCursor.getPartition());
    Assert.assertEquals("000001", nakadiCursor.getOffset());
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) EventTypeCache(org.zalando.nakadi.repository.db.EventTypeCache) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) Cursor(org.zalando.nakadi.view.Cursor) CursorConverter(org.zalando.nakadi.service.CursorConverter) Test(org.junit.Test)

Example 8 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class TopicRepositoryHolderTest method testLockingWhileRepoCreation.

@Test(timeout = 5000L)
public void testLockingWhileRepoCreation() throws InterruptedException {
    final Storage storage = new Storage("1", Storage.Type.KAFKA);
    final TopicRepository[] repos = new TopicRepository[10];
    final AtomicInteger callCount = new AtomicInteger(0);
    final CountDownLatch allowCreation = new CountDownLatch(1);
    final CountDownLatch allRequested = new CountDownLatch(repos.length);
    final CountDownLatch allCreated = new CountDownLatch(repos.length);
    final TopicRepositoryCreator creator = new TopicRepositoryCreator() {

        @Override
        public TopicRepository createTopicRepository(final Storage storage) throws TopicRepositoryException {
            try {
                allowCreation.await();
            } catch (InterruptedException ignore) {
            }
            callCount.incrementAndGet();
            return mock(TopicRepository.class);
        }

        @Override
        public Timeline.StoragePosition createStoragePosition(final List<NakadiCursor> offsets) throws NakadiRuntimeException {
            return null;
        }

        @Override
        public Storage.Type getSupportedStorageType() {
            return Storage.Type.KAFKA;
        }
    };
    final TopicRepositoryHolder holder = new TopicRepositoryHolder(creator);
    for (int i = 0; i < repos.length; ++i) {
        final int currentIdx = i;
        new Thread(() -> {
            allRequested.countDown();
            repos[currentIdx] = holder.getTopicRepository(storage);
            allCreated.countDown();
        }).start();
    }
    allRequested.await();
    Thread.sleep(100L);
    Stream.of(repos).forEach(Assert::assertNull);
    allowCreation.countDown();
    allCreated.await();
    Stream.of(repos).forEach(Assert::assertNotNull);
    Assert.assertEquals(1, Stream.of(repos).collect(Collectors.toSet()).size());
    Assert.assertEquals(1, callCount.get());
}
Also used : CountDownLatch(java.util.concurrent.CountDownLatch) Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) Assert(org.junit.Assert) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) List(java.util.List) Test(org.junit.Test)

Example 9 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class TopicRepositoryHolderTest method testGetTopicRepositoryNoSuchType.

@Test(expected = TopicRepositoryException.class)
public void testGetTopicRepositoryNoSuchType() {
    final TopicRepositoryHolder topicRepositoryHolder = new TopicRepositoryHolder(new TestTopicRepository());
    final Storage storage = new Storage();
    topicRepositoryHolder.getTopicRepository(storage);
}
Also used : Storage(org.zalando.nakadi.domain.Storage) Test(org.junit.Test)

Example 10 with Storage

use of org.zalando.nakadi.domain.Storage in project nakadi by zalando.

the class CursorOperationsService method cursorsLag.

public List<NakadiCursorLag> cursorsLag(final String eventTypeName, final List<NakadiCursor> cursors) throws InvalidCursorOperation {
    try {
        final List<Timeline> timelines = timelineService.getActiveTimelinesOrdered(eventTypeName);
        // Next 2 calls could be optimized to 1 storage call, instead of possible 2 calls.
        // But it is simpler not to do anything, cause timelines are not switched every day and almost all the time
        // (except retention time after switch) there will be only 1 active timeline, and this option is covered.
        final List<PartitionStatistics> oldestStats = getStatsForTimeline(timelines.get(0));
        final List<PartitionStatistics> newestStats = timelines.size() == 1 ? oldestStats : getStatsForTimeline(timelines.get(timelines.size() - 1));
        return cursors.stream().map(c -> {
            final PartitionStatistics oldestStat = oldestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            NakadiCursor newestPosition = newestStats.stream().filter(item -> item.getPartition().equalsIgnoreCase(c.getPartition())).map(PartitionEndStatistics::getLast).findAny().orElseThrow(() -> new InvalidCursorOperation(PARTITION_NOT_FOUND));
            // it
            while (numberOfEventsBeforeCursor(newestPosition) == -1) {
                final int prevOrder = newestPosition.getTimeline().getOrder() - 1;
                final Timeline prevTimeline = timelines.stream().filter(t -> t.getOrder() == prevOrder).findAny().orElse(null);
                if (null == prevTimeline) {
                    break;
                }
                // We moved back, so timeline definitely have latest position set
                newestPosition = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, newestPosition.getPartition());
            }
            // calls (in case of kafka)
            return new NakadiCursorLag(oldestStat.getFirst(), newestPosition, calculateDistance(c, newestPosition));
        }).collect(Collectors.toList());
    } catch (final NakadiException e) {
        throw new MyNakadiRuntimeException1("error", e);
    }
}
Also used : Storage(org.zalando.nakadi.domain.Storage) Logger(org.slf4j.Logger) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) LoggerFactory(org.slf4j.LoggerFactory) PartitionEndStatistics(org.zalando.nakadi.domain.PartitionEndStatistics) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) Collectors(java.util.stream.Collectors) KafkaCursor(org.zalando.nakadi.repository.kafka.KafkaCursor) TIMELINE_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.TIMELINE_NOT_FOUND) List(java.util.List) UnknownStorageTypeException(org.zalando.nakadi.exceptions.runtime.UnknownStorageTypeException) Timeline(org.zalando.nakadi.domain.Timeline) PARTITION_NOT_FOUND(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.PARTITION_NOT_FOUND) Service(org.springframework.stereotype.Service) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) CURSORS_WITH_DIFFERENT_PARTITION(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation.Reason.CURSORS_WITH_DIFFERENT_PARTITION) Collections(java.util.Collections) Timeline(org.zalando.nakadi.domain.Timeline) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) ShiftedNakadiCursor(org.zalando.nakadi.domain.ShiftedNakadiCursor) MyNakadiRuntimeException1(org.zalando.nakadi.exceptions.runtime.MyNakadiRuntimeException1) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) InvalidCursorOperation(org.zalando.nakadi.exceptions.runtime.InvalidCursorOperation) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag) NakadiException(org.zalando.nakadi.exceptions.NakadiException)

Aggregations

Storage (org.zalando.nakadi.domain.Storage)31 Test (org.junit.Test)16 Timeline (org.zalando.nakadi.domain.Timeline)14 NakadiCursor (org.zalando.nakadi.domain.NakadiCursor)10 PartitionStatistics (org.zalando.nakadi.domain.PartitionStatistics)7 TopicRepository (org.zalando.nakadi.repository.TopicRepository)7 Date (java.util.Date)6 DefaultStorage (org.zalando.nakadi.domain.DefaultStorage)6 List (java.util.List)4 EventType (org.zalando.nakadi.domain.EventType)4 ServiceUnavailableException (org.zalando.nakadi.exceptions.ServiceUnavailableException)4 IOException (java.io.IOException)3 Collectors (java.util.stream.Collectors)3 DuplicatedStorageException (org.zalando.nakadi.exceptions.runtime.DuplicatedStorageException)3 TimelineService (org.zalando.nakadi.service.timeline.TimelineService)3 Cursor (org.zalando.nakadi.view.Cursor)3 SubscriptionCursorWithoutToken (org.zalando.nakadi.view.SubscriptionCursorWithoutToken)3 Collections (java.util.Collections)2 Optional (java.util.Optional)2 Logger (org.slf4j.Logger)2