Search in sources :

Example 46 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class KafkaTopicRepositoryTest method canLoadPartitionStatistics.

@Test
public void canLoadPartitionStatistics() throws Exception {
    final Timeline t1 = mock(Timeline.class);
    when(t1.getTopic()).thenReturn(MY_TOPIC);
    final Timeline t2 = mock(Timeline.class);
    when(t2.getTopic()).thenReturn(ANOTHER_TOPIC);
    final ImmutableList<Timeline> timelines = ImmutableList.of(t1, t2);
    final List<PartitionStatistics> stats = kafkaTopicRepository.loadTopicStatistics(timelines);
    final Set<PartitionStatistics> expected = PARTITIONS.stream().map(p -> {
        final Timeline timeline = p.topic.equals(MY_TOPIC) ? t1 : t2;
        return new KafkaPartitionStatistics(timeline, p.partition, p.earliestOffset, p.latestOffset - 1);
    }).collect(Collectors.toSet());
    assertThat(newHashSet(stats), equalTo(expected));
}
Also used : EventPublishingException(org.zalando.nakadi.exceptions.EventPublishingException) GetChildrenBuilder(org.apache.curator.framework.api.GetChildrenBuilder) KafkaException(org.apache.kafka.common.KafkaException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) Assert.assertThat(org.junit.Assert.assertThat) Future(java.util.concurrent.Future) Arrays.asList(java.util.Arrays.asList) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Sets.newHashSet(com.google.common.collect.Sets.newHashSet) Assert.fail(org.junit.Assert.fail) Matchers.anyVararg(org.mockito.Matchers.anyVararg) Consumer(org.apache.kafka.clients.consumer.Consumer) ZooKeeperHolder(org.zalando.nakadi.repository.zookeeper.ZooKeeperHolder) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) Set(java.util.Set) PartitionInfo(org.apache.kafka.common.PartitionInfo) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) Collectors(java.util.stream.Collectors) Matchers.any(org.mockito.Matchers.any) List(java.util.List) CuratorFramework(org.apache.curator.framework.CuratorFramework) TestUtils.buildTimelineWithTopic(org.zalando.nakadi.utils.TestUtils.buildTimelineWithTopic) Timeline(org.zalando.nakadi.domain.Timeline) ZookeeperSettings(org.zalando.nakadi.repository.zookeeper.ZookeeperSettings) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) BatchItem(org.zalando.nakadi.domain.BatchItem) Node(org.apache.kafka.common.Node) Matchers.is(org.hamcrest.Matchers.is) Callback(org.apache.kafka.clients.producer.Callback) Mockito.mock(org.mockito.Mockito.mock) ProducerRecord(org.apache.kafka.clients.producer.ProducerRecord) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) NakadiSettings(org.zalando.nakadi.config.NakadiSettings) Cursor(org.zalando.nakadi.view.Cursor) Matchers.anyString(org.mockito.Matchers.anyString) ArrayList(java.util.ArrayList) UUIDGenerator(org.zalando.nakadi.util.UUIDGenerator) HashSet(java.util.HashSet) KafkaProducer(org.apache.kafka.clients.producer.KafkaProducer) ImmutableList(com.google.common.collect.ImmutableList) LinkedList(java.util.LinkedList) TimeoutException(org.apache.kafka.common.errors.TimeoutException) PartitionEndStatistics(org.zalando.nakadi.domain.PartitionEndStatistics) Test(org.junit.Test) BufferExhaustedException(org.apache.kafka.clients.producer.BufferExhaustedException) Mockito.when(org.mockito.Mockito.when) Mockito(org.mockito.Mockito) Collectors.toList(java.util.stream.Collectors.toList) EventPublishingStatus(org.zalando.nakadi.domain.EventPublishingStatus) CursorError(org.zalando.nakadi.domain.CursorError) Assert(org.junit.Assert) Collections(java.util.Collections) Timeline(org.zalando.nakadi.domain.Timeline) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) Test(org.junit.Test)

Example 47 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class PartitionsControllerTest method mockCursorLag.

private List<NakadiCursorLag> mockCursorLag() {
    final Timeline timeline = mock(Timeline.class);
    when(timeline.getStorage()).thenReturn(new Storage("ccc", Storage.Type.KAFKA));
    final NakadiCursorLag lag = mock(NakadiCursorLag.class);
    final NakadiCursor firstCursor = NakadiCursor.of(timeline, "0", "0");
    final NakadiCursor lastCursor = NakadiCursor.of(timeline, "0", "1");
    when(lag.getLag()).thenReturn(42L);
    when(lag.getPartition()).thenReturn("0");
    when(lag.getFirstCursor()).thenReturn(firstCursor);
    when(lag.getLastCursor()).thenReturn(lastCursor);
    return Lists.newArrayList(lag);
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) TestUtils.buildTimeline(org.zalando.nakadi.utils.TestUtils.buildTimeline) Storage(org.zalando.nakadi.domain.Storage) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) NakadiCursorLag(org.zalando.nakadi.domain.NakadiCursorLag)

Example 48 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class TimelinesControllerTest method whenGetTimelinesThenOk.

@Test
public void whenGetTimelinesThenOk() throws Exception {
    final Storage kafkaStorage = StoragesControllerTest.createKafkaStorage("deafult");
    final ImmutableList<Timeline> timelines = ImmutableList.of(Timeline.createTimeline("event_type", 0, kafkaStorage, "topic", new Date()), Timeline.createTimeline("event_type_1", 1, kafkaStorage, "topic_1", new Date()));
    Mockito.when(timelineService.getTimelines(Mockito.any())).thenReturn(timelines);
    final List<TimelineView> timelineViews = timelines.stream().map(TimelineView::new).collect(Collectors.toList());
    mockMvc.perform(MockMvcRequestBuilders.get("/event-types/event_type/timelines").contentType(MediaType.APPLICATION_JSON).principal(PrincipalMockFactory.mockPrincipal("nakadi"))).andExpect(MockMvcResultMatchers.status().isOk()).andExpect(MockMvcResultMatchers.content().json(TestUtils.OBJECT_MAPPER.writeValueAsString(timelineViews)));
}
Also used : TimelineView(org.zalando.nakadi.view.TimelineView) Timeline(org.zalando.nakadi.domain.Timeline) Storage(org.zalando.nakadi.domain.Storage) Date(java.util.Date) Test(org.junit.Test)

Example 49 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class TimelineService method deleteAllTimelinesForEventType.

public Multimap<TopicRepository, String> deleteAllTimelinesForEventType(final String eventTypeName) throws TimelineException, NotFoundException, InternalNakadiException, NoSuchEventTypeException {
    LOG.info("Deleting all timelines for event type {}", eventTypeName);
    final Multimap<TopicRepository, String> topicsToDelete = ArrayListMultimap.create();
    for (final Timeline timeline : getAllTimelinesOrdered(eventTypeName)) {
        if (!timeline.isDeleted()) {
            topicsToDelete.put(getTopicRepository(timeline), timeline.getTopic());
        }
        timelineDbRepository.deleteTimeline(timeline.getId());
    }
    return topicsToDelete;
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) TopicRepository(org.zalando.nakadi.repository.TopicRepository)

Example 50 with Timeline

use of org.zalando.nakadi.domain.Timeline in project nakadi by zalando.

the class TimelineService method switchTimelines.

private void switchTimelines(final Timeline activeTimeline, final Timeline nextTimeline) throws InconsistentStateException, RepositoryProblemException, TimelineException, ConflictException {
    LOG.info("Switching timelines from {} to {}", activeTimeline, nextTimeline);
    try {
        timelineSync.startTimelineUpdate(activeTimeline.getEventType(), nakadiSettings.getTimelineWaitTimeoutMs());
    } catch (final InterruptedException ie) {
        Thread.currentThread().interrupt();
        throw new TimelineException("Failed to switch timeline for: " + activeTimeline.getEventType());
    } catch (final IllegalStateException ie) {
        throw new ConflictException("Timeline is already being created for: " + activeTimeline.getEventType(), ie);
    }
    try {
        transactionTemplate.execute(status -> {
            timelineDbRepository.createTimeline(nextTimeline);
            nextTimeline.setSwitchedAt(new Date());
            final Timeline.StoragePosition sp = topicRepositoryHolder.createStoragePosition(activeTimeline);
            activeTimeline.setLatestPosition(sp);
            scheduleTimelineCleanup(activeTimeline);
            timelineDbRepository.updateTimelime(activeTimeline);
            timelineDbRepository.updateTimelime(nextTimeline);
            return null;
        });
    } catch (final TransactionException tx) {
        LOG.error(tx.getMessage(), tx);
        throw new TimelineException("Failed to create timeline in DB for: " + activeTimeline.getEventType(), tx);
    } finally {
        finishTimelineUpdate(activeTimeline.getEventType());
    }
}
Also used : Timeline(org.zalando.nakadi.domain.Timeline) TransactionException(org.springframework.transaction.TransactionException) ConflictException(org.zalando.nakadi.exceptions.ConflictException) Date(java.util.Date) TimelineException(org.zalando.nakadi.exceptions.TimelineException) DuplicatedTimelineException(org.zalando.nakadi.exceptions.runtime.DuplicatedTimelineException)

Aggregations

Timeline (org.zalando.nakadi.domain.Timeline)74 Test (org.junit.Test)39 NakadiCursor (org.zalando.nakadi.domain.NakadiCursor)33 ShiftedNakadiCursor (org.zalando.nakadi.domain.ShiftedNakadiCursor)19 TopicRepository (org.zalando.nakadi.repository.TopicRepository)17 Date (java.util.Date)14 EventType (org.zalando.nakadi.domain.EventType)14 List (java.util.List)13 PartitionStatistics (org.zalando.nakadi.domain.PartitionStatistics)13 Storage (org.zalando.nakadi.domain.Storage)13 Collectors (java.util.stream.Collectors)12 InvalidCursorException (org.zalando.nakadi.exceptions.InvalidCursorException)12 ArrayList (java.util.ArrayList)9 Collections (java.util.Collections)9 NakadiSettings (org.zalando.nakadi.config.NakadiSettings)9 InternalNakadiException (org.zalando.nakadi.exceptions.InternalNakadiException)9 Optional (java.util.Optional)8 Logger (org.slf4j.Logger)8 LoggerFactory (org.slf4j.LoggerFactory)8 NakadiException (org.zalando.nakadi.exceptions.NakadiException)8