Search in sources :

Example 1 with EventStream

use of org.zalando.nakadi.service.EventStream in project nakadi by zalando.

the class EventStreamControllerTest method reportCurrentNumberOfConsumers.

@Test
public void reportCurrentNumberOfConsumers() throws Exception {
    when(eventTypeRepository.findByName(TEST_EVENT_TYPE_NAME)).thenReturn(EVENT_TYPE);
    final EventStream eventStream = mock(EventStream.class);
    // block to simulate the streaming until thread is interrupted
    Mockito.doAnswer(invocation -> {
        while (!Thread.interrupted()) {
            Thread.sleep(100);
        }
        return null;
    }).when(eventStream).streamEvents(any(), any());
    when(eventStreamFactoryMock.createEventStream(any(), any(), any(), any())).thenReturn(eventStream);
    // "connect" to the server
    final StreamingResponseBody responseBody = createStreamingResponseBody();
    final LinkedList<Thread> clients = new LinkedList<>();
    final Counter counter = metricRegistry.counter(metricNameFor(TEST_EVENT_TYPE_NAME, EventStreamController.CONSUMERS_COUNT_METRIC_NAME));
    // create clients...
    for (int i = 0; i < 3; i++) {
        final Thread client = new Thread(() -> {
            try {
                responseBody.writeTo(new ByteArrayOutputStream());
            } catch (final IOException e) {
                throw new RuntimeException(e);
            }
        });
        client.start();
        clients.add(client);
        TestUtils.waitFor(() -> assertThat(counter.getCount(), equalTo((long) clients.size())), TimeUnit.SECONDS.toMillis(5));
    }
    // ...and disconnect them one by one
    while (!clients.isEmpty()) {
        final Thread client = clients.pop();
        client.interrupt();
        client.join();
        assertThat(counter.getCount(), equalTo((long) clients.size()));
    }
}
Also used : Counter(com.codahale.metrics.Counter) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) EventStream(org.zalando.nakadi.service.EventStream) ByteArrayOutputStream(java.io.ByteArrayOutputStream) IOException(java.io.IOException) LinkedList(java.util.LinkedList) Test(org.junit.Test)

Example 2 with EventStream

use of org.zalando.nakadi.service.EventStream in project nakadi by zalando.

the class EventStreamController method streamEvents.

@RequestMapping(value = "/event-types/{name}/events", method = RequestMethod.GET)
public StreamingResponseBody streamEvents(@PathVariable("name") final String eventTypeName, @Nullable @RequestParam(value = "batch_limit", required = false) final Integer batchLimit, @Nullable @RequestParam(value = "stream_limit", required = false) final Integer streamLimit, @Nullable @RequestParam(value = "batch_flush_timeout", required = false) final Integer batchTimeout, @Nullable @RequestParam(value = "stream_timeout", required = false) final Integer streamTimeout, @Nullable @RequestParam(value = "stream_keep_alive_limit", required = false) final Integer streamKeepAliveLimit, @Nullable @RequestHeader(name = "X-nakadi-cursors", required = false) final String cursorsStr, final HttpServletRequest request, final HttpServletResponse response, final Client client) {
    final String flowId = FlowIdUtils.peek();
    return outputStream -> {
        FlowIdUtils.push(flowId);
        if (blacklistService.isConsumptionBlocked(eventTypeName, client.getClientId())) {
            writeProblemResponse(response, outputStream, Problem.valueOf(Response.Status.FORBIDDEN, "Application or event type is blocked"));
            return;
        }
        final AtomicBoolean connectionReady = closedConnectionsCrutch.listenForConnectionClose(request);
        Counter consumerCounter = null;
        EventStream eventStream = null;
        List<ConnectionSlot> connectionSlots = ImmutableList.of();
        final AtomicBoolean needCheckAuthorization = new AtomicBoolean(false);
        LOG.info("[X-NAKADI-CURSORS] \"{}\" {}", eventTypeName, Optional.ofNullable(cursorsStr).orElse("-"));
        try (Closeable ignore = eventTypeChangeListener.registerListener(et -> needCheckAuthorization.set(true), Collections.singletonList(eventTypeName))) {
            final EventType eventType = eventTypeRepository.findByName(eventTypeName);
            authorizeStreamRead(eventTypeName);
            // validate parameters
            final EventStreamConfig streamConfig = EventStreamConfig.builder().withBatchLimit(batchLimit).withStreamLimit(streamLimit).withBatchTimeout(batchTimeout).withStreamTimeout(streamTimeout).withStreamKeepAliveLimit(streamKeepAliveLimit).withEtName(eventTypeName).withConsumingClient(client).withCursors(getStreamingStart(eventType, cursorsStr)).withMaxMemoryUsageBytes(maxMemoryUsageBytes).build();
            // acquire connection slots to limit the number of simultaneous connections from one client
            if (featureToggleService.isFeatureEnabled(LIMIT_CONSUMERS_NUMBER)) {
                final List<String> partitions = streamConfig.getCursors().stream().map(NakadiCursor::getPartition).collect(Collectors.toList());
                connectionSlots = consumerLimitingService.acquireConnectionSlots(client.getClientId(), eventTypeName, partitions);
            }
            consumerCounter = metricRegistry.counter(metricNameFor(eventTypeName, CONSUMERS_COUNT_METRIC_NAME));
            consumerCounter.inc();
            final String kafkaQuotaClientId = getKafkaQuotaClientId(eventTypeName, client);
            response.setStatus(HttpStatus.OK.value());
            response.setHeader("Warning", "299 - nakadi - the Low-level API is deprecated and will " + "be removed from a future release. Please consider migrating to the Subscriptions API.");
            response.setContentType("application/x-json-stream");
            final EventConsumer eventConsumer = timelineService.createEventConsumer(kafkaQuotaClientId, streamConfig.getCursors());
            final String bytesFlushedMetricName = MetricUtils.metricNameForLoLAStream(client.getClientId(), eventTypeName);
            final Meter bytesFlushedMeter = this.streamMetrics.meter(bytesFlushedMetricName);
            eventStream = eventStreamFactory.createEventStream(outputStream, eventConsumer, streamConfig, bytesFlushedMeter);
            // Flush status code to client
            outputStream.flush();
            eventStream.streamEvents(connectionReady, () -> {
                if (needCheckAuthorization.getAndSet(false)) {
                    authorizeStreamRead(eventTypeName);
                }
            });
        } catch (final UnparseableCursorException e) {
            LOG.debug("Incorrect syntax of X-nakadi-cursors header: {}. Respond with BAD_REQUEST.", e.getCursors(), e);
            writeProblemResponse(response, outputStream, BAD_REQUEST, e.getMessage());
        } catch (final NoSuchEventTypeException e) {
            writeProblemResponse(response, outputStream, NOT_FOUND, "topic not found");
        } catch (final NoConnectionSlotsException e) {
            LOG.debug("Connection creation failed due to exceeding max connection count");
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final NakadiException e) {
            LOG.error("Error while trying to stream events.", e);
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final InvalidCursorException e) {
            writeProblemResponse(response, outputStream, PRECONDITION_FAILED, e.getMessage());
        } catch (final AccessDeniedException e) {
            writeProblemResponse(response, outputStream, FORBIDDEN, e.explain());
        } catch (final Exception e) {
            LOG.error("Error while trying to stream events. Respond with INTERNAL_SERVER_ERROR.", e);
            writeProblemResponse(response, outputStream, INTERNAL_SERVER_ERROR, e.getMessage());
        } finally {
            connectionReady.set(false);
            consumerLimitingService.releaseConnectionSlots(connectionSlots);
            if (consumerCounter != null) {
                consumerCounter.dec();
            }
            if (eventStream != null) {
                eventStream.close();
            }
            try {
                outputStream.flush();
            } finally {
                outputStream.close();
            }
        }
    };
}
Also used : PathVariable(org.springframework.web.bind.annotation.PathVariable) RequestParam(org.springframework.web.bind.annotation.RequestParam) FlowIdUtils(org.zalando.nakadi.util.FlowIdUtils) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) Problem(org.zalando.problem.Problem) Map(java.util.Map) Counter(com.codahale.metrics.Counter) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TypeReference(com.fasterxml.jackson.core.type.TypeReference) FeatureToggleService(org.zalando.nakadi.service.FeatureToggleService) EventTypeChangeListener(org.zalando.nakadi.service.EventTypeChangeListener) BAD_REQUEST(javax.ws.rs.core.Response.Status.BAD_REQUEST) ConsumerLimitingService(org.zalando.nakadi.service.ConsumerLimitingService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) ConnectionSlot(org.zalando.nakadi.service.ConnectionSlot) NOT_FOUND(javax.ws.rs.core.Response.Status.NOT_FOUND) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) RequestMethod(org.springframework.web.bind.annotation.RequestMethod) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) BlacklistService(org.zalando.nakadi.service.BlacklistService) RestController(org.springframework.web.bind.annotation.RestController) Collectors(java.util.stream.Collectors) List(java.util.List) Response(javax.ws.rs.core.Response) Timeline(org.zalando.nakadi.domain.Timeline) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) Optional(java.util.Optional) RequestHeader(org.springframework.web.bind.annotation.RequestHeader) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) Client(org.zalando.nakadi.security.Client) Storage(org.zalando.nakadi.domain.Storage) ClosedConnectionsCrutch(org.zalando.nakadi.service.ClosedConnectionsCrutch) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) RequestMapping(org.springframework.web.bind.annotation.RequestMapping) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Cursor(org.zalando.nakadi.view.Cursor) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) Meter(com.codahale.metrics.Meter) HttpServletRequest(javax.servlet.http.HttpServletRequest) ImmutableList(com.google.common.collect.ImmutableList) EventStreamFactory(org.zalando.nakadi.service.EventStreamFactory) Qualifier(org.springframework.beans.factory.annotation.Qualifier) PRECONDITION_FAILED(javax.ws.rs.core.Response.Status.PRECONDITION_FAILED) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) EventType(org.zalando.nakadi.domain.EventType) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) INTERNAL_SERVER_ERROR(javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) HttpServletResponse(javax.servlet.http.HttpServletResponse) MetricUtils.metricNameFor(org.zalando.nakadi.metrics.MetricUtils.metricNameFor) IOException(java.io.IOException) FORBIDDEN(javax.ws.rs.core.Response.Status.FORBIDDEN) EventStream(org.zalando.nakadi.service.EventStream) AuthorizationValidator(org.zalando.nakadi.service.AuthorizationValidator) HttpStatus(org.springframework.http.HttpStatus) EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventTypeRepository(org.zalando.nakadi.repository.EventTypeRepository) Closeable(java.io.Closeable) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) LIMIT_CONSUMERS_NUMBER(org.zalando.nakadi.service.FeatureToggleService.Feature.LIMIT_CONSUMERS_NUMBER) CursorError(org.zalando.nakadi.domain.CursorError) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetricUtils(org.zalando.nakadi.metrics.MetricUtils) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) Collections(java.util.Collections) CursorConverter(org.zalando.nakadi.service.CursorConverter) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) EventType(org.zalando.nakadi.domain.EventType) Meter(com.codahale.metrics.Meter) Closeable(java.io.Closeable) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) IOException(java.io.IOException) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) EventConsumer(org.zalando.nakadi.repository.EventConsumer) Counter(com.codahale.metrics.Counter) EventStream(org.zalando.nakadi.service.EventStream) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) RequestMapping(org.springframework.web.bind.annotation.RequestMapping)

Example 3 with EventStream

use of org.zalando.nakadi.service.EventStream in project nakadi by zalando.

the class EventStreamControllerTest method whenNoParamsThenDefaultsAreUsed.

@Test
@SuppressWarnings("unchecked")
public void whenNoParamsThenDefaultsAreUsed() throws Exception {
    final ArgumentCaptor<EventStreamConfig> configCaptor = ArgumentCaptor.forClass(EventStreamConfig.class);
    final EventConsumer.LowLevelConsumer eventConsumerMock = mock(EventConsumer.LowLevelConsumer.class);
    when(topicRepositoryMock.createEventConsumer(any(), any())).thenReturn(eventConsumerMock);
    final EventStream eventStreamMock = mock(EventStream.class);
    when(eventStreamFactoryMock.createEventStream(any(), any(), configCaptor.capture(), any())).thenReturn(eventStreamMock);
    when(eventTypeRepository.findByName(TEST_EVENT_TYPE_NAME)).thenReturn(EVENT_TYPE);
    mockMvc.perform(get(String.format("/event-types/%s/events", TEST_EVENT_TYPE_NAME)).header("X-nakadi-cursors", "[{\"partition\":\"0\",\"offset\":\"000000000000000000\"}]")).andExpect(status().isOk());
    // we have to retry here as mockMvc exits at the very beginning, before the body starts streaming
    TestUtils.waitFor(() -> {
        final EventStreamConfig actualConfig = configCaptor.getValue();
        assertThat(actualConfig.getBatchLimit(), equalTo(1));
        assertThat(actualConfig.getBatchTimeout(), equalTo(30));
        assertThat(actualConfig.getCursors(), equalTo(ImmutableList.of(NakadiCursor.of(timeline, "0", "000000000000000000"))));
        assertThat(actualConfig.getStreamKeepAliveLimit(), equalTo(0));
        assertThat(actualConfig.getStreamLimit(), equalTo(0));
        assertThat(actualConfig.getStreamTimeout(), greaterThanOrEqualTo(EventStreamConfig.MAX_STREAM_TIMEOUT - 1200));
        assertThat(actualConfig.getStreamTimeout(), lessThanOrEqualTo(EventStreamConfig.MAX_STREAM_TIMEOUT));
    }, 2000, 50, MockitoException.class);
}
Also used : EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventStream(org.zalando.nakadi.service.EventStream) Test(org.junit.Test)

Example 4 with EventStream

use of org.zalando.nakadi.service.EventStream in project nakadi by zalando.

the class EventStreamControllerTest method whenNoCursorsThenLatestOffsetsAreUsed.

@Test
public void whenNoCursorsThenLatestOffsetsAreUsed() throws NakadiException, IOException, InvalidCursorException {
    when(eventTypeRepository.findByName(TEST_EVENT_TYPE_NAME)).thenReturn(EVENT_TYPE);
    final List<PartitionStatistics> tps2 = ImmutableList.of(new KafkaPartitionStatistics(timeline, 0, 0, 87), new KafkaPartitionStatistics(timeline, 1, 0, 34));
    when(timelineService.getActiveTimeline(any(EventType.class))).thenReturn(timeline);
    when(topicRepositoryMock.loadTopicStatistics(eq(Collections.singletonList(timeline)))).thenReturn(tps2);
    final ArgumentCaptor<EventStreamConfig> configCaptor = ArgumentCaptor.forClass(EventStreamConfig.class);
    final EventStream eventStreamMock = mock(EventStream.class);
    when(eventStreamFactoryMock.createEventStream(any(), any(), configCaptor.capture(), any())).thenReturn(eventStreamMock);
    final StreamingResponseBody responseBody = createStreamingResponseBody(1, 0, 1, 1, 0, null);
    responseBody.writeTo(new ByteArrayOutputStream());
    final EventStreamConfig streamConfig = configCaptor.getValue();
    assertThat(streamConfig.getCursors(), equalTo(tps2.stream().map(PartitionStatistics::getLast).collect(Collectors.toList())));
}
Also used : EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) KafkaPartitionStatistics(org.zalando.nakadi.repository.kafka.KafkaPartitionStatistics) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) EventType(org.zalando.nakadi.domain.EventType) KafkaPartitionStatistics(org.zalando.nakadi.repository.kafka.KafkaPartitionStatistics) EventStream(org.zalando.nakadi.service.EventStream) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Test(org.junit.Test)

Example 5 with EventStream

use of org.zalando.nakadi.service.EventStream in project nakadi by zalando.

the class EventStreamControllerTest method whenNormalCaseThenParametersArePassedToConfigAndStreamStarted.

@Test
public void whenNormalCaseThenParametersArePassedToConfigAndStreamStarted() throws Exception {
    final EventConsumer eventConsumerMock = mock(EventConsumer.class);
    when(eventTypeRepository.findByName(TEST_EVENT_TYPE_NAME)).thenReturn(EVENT_TYPE);
    when(timelineService.createEventConsumer(eq(KAFKA_CLIENT_ID), eq(ImmutableList.of(NakadiCursor.of(timeline, "0", "000000000000000000"))))).thenReturn(eventConsumerMock);
    when(timelineService.getActiveTimeline(eq(EVENT_TYPE))).thenReturn(timeline);
    final ArgumentCaptor<Integer> statusCaptor = getStatusCaptor();
    final ArgumentCaptor<String> contentTypeCaptor = getContentTypeCaptor();
    final ArgumentCaptor<EventStreamConfig> configCaptor = ArgumentCaptor.forClass(EventStreamConfig.class);
    final EventStream eventStreamMock = mock(EventStream.class);
    when(eventStreamFactoryMock.createEventStream(any(), any(), configCaptor.capture(), any())).thenReturn(eventStreamMock);
    final StreamingResponseBody responseBody = createStreamingResponseBody(1, 2, 3, 4, 5, "[{\"partition\":\"0\",\"offset\":\"000000000000000000\"}]");
    final OutputStream outputStream = mock(OutputStream.class);
    responseBody.writeTo(outputStream);
    final EventStreamConfig streamConfig = configCaptor.getValue();
    assertThat(streamConfig, equalTo(EventStreamConfig.builder().withCursors(ImmutableList.of(NakadiCursor.of(timeline, "0", "000000000000000000"))).withBatchLimit(1).withStreamLimit(2).withBatchTimeout(3).withStreamTimeout(4).withStreamKeepAliveLimit(5).build()));
    assertThat(statusCaptor.getValue(), equalTo(HttpStatus.OK.value()));
    assertThat(contentTypeCaptor.getValue(), equalTo("application/x-json-stream"));
    verify(timelineService, times(1)).createEventConsumer(eq(KAFKA_CLIENT_ID), eq(ImmutableList.of(NakadiCursor.of(timeline, "0", "000000000000000000"))));
    verify(eventStreamFactoryMock, times(1)).createEventStream(eq(outputStream), eq(eventConsumerMock), eq(streamConfig), any());
    verify(eventStreamMock, times(1)).streamEvents(any(), any());
    verify(outputStream, times(2)).flush();
    verify(outputStream, times(1)).close();
}
Also used : EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) EventStream(org.zalando.nakadi.service.EventStream) ByteArrayOutputStream(java.io.ByteArrayOutputStream) OutputStream(java.io.OutputStream) Test(org.junit.Test)

Aggregations

EventStream (org.zalando.nakadi.service.EventStream)5 Test (org.junit.Test)4 StreamingResponseBody (org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody)4 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3 EventStreamConfig (org.zalando.nakadi.service.EventStreamConfig)3 Counter (com.codahale.metrics.Counter)2 IOException (java.io.IOException)2 OutputStream (java.io.OutputStream)2 EventConsumer (org.zalando.nakadi.repository.EventConsumer)2 Meter (com.codahale.metrics.Meter)1 MetricRegistry (com.codahale.metrics.MetricRegistry)1 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 ImmutableList (com.google.common.collect.ImmutableList)1 Closeable (java.io.Closeable)1 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 LinkedList (java.util.LinkedList)1 List (java.util.List)1