Search in sources :

Example 1 with LIMIT_CONSUMERS_NUMBER

use of org.zalando.nakadi.service.FeatureToggleService.Feature.LIMIT_CONSUMERS_NUMBER in project nakadi by zalando.

the class EventStreamController method streamEvents.

@RequestMapping(value = "/event-types/{name}/events", method = RequestMethod.GET)
public StreamingResponseBody streamEvents(@PathVariable("name") final String eventTypeName, @Nullable @RequestParam(value = "batch_limit", required = false) final Integer batchLimit, @Nullable @RequestParam(value = "stream_limit", required = false) final Integer streamLimit, @Nullable @RequestParam(value = "batch_flush_timeout", required = false) final Integer batchTimeout, @Nullable @RequestParam(value = "stream_timeout", required = false) final Integer streamTimeout, @Nullable @RequestParam(value = "stream_keep_alive_limit", required = false) final Integer streamKeepAliveLimit, @Nullable @RequestHeader(name = "X-nakadi-cursors", required = false) final String cursorsStr, final HttpServletRequest request, final HttpServletResponse response, final Client client) {
    final String flowId = FlowIdUtils.peek();
    return outputStream -> {
        FlowIdUtils.push(flowId);
        if (blacklistService.isConsumptionBlocked(eventTypeName, client.getClientId())) {
            writeProblemResponse(response, outputStream, Problem.valueOf(Response.Status.FORBIDDEN, "Application or event type is blocked"));
            return;
        }
        final AtomicBoolean connectionReady = closedConnectionsCrutch.listenForConnectionClose(request);
        Counter consumerCounter = null;
        EventStream eventStream = null;
        List<ConnectionSlot> connectionSlots = ImmutableList.of();
        final AtomicBoolean needCheckAuthorization = new AtomicBoolean(false);
        LOG.info("[X-NAKADI-CURSORS] \"{}\" {}", eventTypeName, Optional.ofNullable(cursorsStr).orElse("-"));
        try (Closeable ignore = eventTypeChangeListener.registerListener(et -> needCheckAuthorization.set(true), Collections.singletonList(eventTypeName))) {
            final EventType eventType = eventTypeRepository.findByName(eventTypeName);
            authorizeStreamRead(eventTypeName);
            // validate parameters
            final EventStreamConfig streamConfig = EventStreamConfig.builder().withBatchLimit(batchLimit).withStreamLimit(streamLimit).withBatchTimeout(batchTimeout).withStreamTimeout(streamTimeout).withStreamKeepAliveLimit(streamKeepAliveLimit).withEtName(eventTypeName).withConsumingClient(client).withCursors(getStreamingStart(eventType, cursorsStr)).withMaxMemoryUsageBytes(maxMemoryUsageBytes).build();
            // acquire connection slots to limit the number of simultaneous connections from one client
            if (featureToggleService.isFeatureEnabled(LIMIT_CONSUMERS_NUMBER)) {
                final List<String> partitions = streamConfig.getCursors().stream().map(NakadiCursor::getPartition).collect(Collectors.toList());
                connectionSlots = consumerLimitingService.acquireConnectionSlots(client.getClientId(), eventTypeName, partitions);
            }
            consumerCounter = metricRegistry.counter(metricNameFor(eventTypeName, CONSUMERS_COUNT_METRIC_NAME));
            consumerCounter.inc();
            final String kafkaQuotaClientId = getKafkaQuotaClientId(eventTypeName, client);
            response.setStatus(HttpStatus.OK.value());
            response.setHeader("Warning", "299 - nakadi - the Low-level API is deprecated and will " + "be removed from a future release. Please consider migrating to the Subscriptions API.");
            response.setContentType("application/x-json-stream");
            final EventConsumer eventConsumer = timelineService.createEventConsumer(kafkaQuotaClientId, streamConfig.getCursors());
            final String bytesFlushedMetricName = MetricUtils.metricNameForLoLAStream(client.getClientId(), eventTypeName);
            final Meter bytesFlushedMeter = this.streamMetrics.meter(bytesFlushedMetricName);
            eventStream = eventStreamFactory.createEventStream(outputStream, eventConsumer, streamConfig, bytesFlushedMeter);
            // Flush status code to client
            outputStream.flush();
            eventStream.streamEvents(connectionReady, () -> {
                if (needCheckAuthorization.getAndSet(false)) {
                    authorizeStreamRead(eventTypeName);
                }
            });
        } catch (final UnparseableCursorException e) {
            LOG.debug("Incorrect syntax of X-nakadi-cursors header: {}. Respond with BAD_REQUEST.", e.getCursors(), e);
            writeProblemResponse(response, outputStream, BAD_REQUEST, e.getMessage());
        } catch (final NoSuchEventTypeException e) {
            writeProblemResponse(response, outputStream, NOT_FOUND, "topic not found");
        } catch (final NoConnectionSlotsException e) {
            LOG.debug("Connection creation failed due to exceeding max connection count");
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final NakadiException e) {
            LOG.error("Error while trying to stream events.", e);
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final InvalidCursorException e) {
            writeProblemResponse(response, outputStream, PRECONDITION_FAILED, e.getMessage());
        } catch (final AccessDeniedException e) {
            writeProblemResponse(response, outputStream, FORBIDDEN, e.explain());
        } catch (final Exception e) {
            LOG.error("Error while trying to stream events. Respond with INTERNAL_SERVER_ERROR.", e);
            writeProblemResponse(response, outputStream, INTERNAL_SERVER_ERROR, e.getMessage());
        } finally {
            connectionReady.set(false);
            consumerLimitingService.releaseConnectionSlots(connectionSlots);
            if (consumerCounter != null) {
                consumerCounter.dec();
            }
            if (eventStream != null) {
                eventStream.close();
            }
            try {
                outputStream.flush();
            } finally {
                outputStream.close();
            }
        }
    };
}
Also used : PathVariable(org.springframework.web.bind.annotation.PathVariable) RequestParam(org.springframework.web.bind.annotation.RequestParam) FlowIdUtils(org.zalando.nakadi.util.FlowIdUtils) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) Problem(org.zalando.problem.Problem) Map(java.util.Map) Counter(com.codahale.metrics.Counter) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TypeReference(com.fasterxml.jackson.core.type.TypeReference) FeatureToggleService(org.zalando.nakadi.service.FeatureToggleService) EventTypeChangeListener(org.zalando.nakadi.service.EventTypeChangeListener) BAD_REQUEST(javax.ws.rs.core.Response.Status.BAD_REQUEST) ConsumerLimitingService(org.zalando.nakadi.service.ConsumerLimitingService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) ConnectionSlot(org.zalando.nakadi.service.ConnectionSlot) NOT_FOUND(javax.ws.rs.core.Response.Status.NOT_FOUND) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) RequestMethod(org.springframework.web.bind.annotation.RequestMethod) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) BlacklistService(org.zalando.nakadi.service.BlacklistService) RestController(org.springframework.web.bind.annotation.RestController) Collectors(java.util.stream.Collectors) List(java.util.List) Response(javax.ws.rs.core.Response) Timeline(org.zalando.nakadi.domain.Timeline) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) Optional(java.util.Optional) RequestHeader(org.springframework.web.bind.annotation.RequestHeader) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) Client(org.zalando.nakadi.security.Client) Storage(org.zalando.nakadi.domain.Storage) ClosedConnectionsCrutch(org.zalando.nakadi.service.ClosedConnectionsCrutch) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) RequestMapping(org.springframework.web.bind.annotation.RequestMapping) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Cursor(org.zalando.nakadi.view.Cursor) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) Meter(com.codahale.metrics.Meter) HttpServletRequest(javax.servlet.http.HttpServletRequest) ImmutableList(com.google.common.collect.ImmutableList) EventStreamFactory(org.zalando.nakadi.service.EventStreamFactory) Qualifier(org.springframework.beans.factory.annotation.Qualifier) PRECONDITION_FAILED(javax.ws.rs.core.Response.Status.PRECONDITION_FAILED) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) EventType(org.zalando.nakadi.domain.EventType) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) INTERNAL_SERVER_ERROR(javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) HttpServletResponse(javax.servlet.http.HttpServletResponse) MetricUtils.metricNameFor(org.zalando.nakadi.metrics.MetricUtils.metricNameFor) IOException(java.io.IOException) FORBIDDEN(javax.ws.rs.core.Response.Status.FORBIDDEN) EventStream(org.zalando.nakadi.service.EventStream) AuthorizationValidator(org.zalando.nakadi.service.AuthorizationValidator) HttpStatus(org.springframework.http.HttpStatus) EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventTypeRepository(org.zalando.nakadi.repository.EventTypeRepository) Closeable(java.io.Closeable) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) LIMIT_CONSUMERS_NUMBER(org.zalando.nakadi.service.FeatureToggleService.Feature.LIMIT_CONSUMERS_NUMBER) CursorError(org.zalando.nakadi.domain.CursorError) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetricUtils(org.zalando.nakadi.metrics.MetricUtils) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) Collections(java.util.Collections) CursorConverter(org.zalando.nakadi.service.CursorConverter) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) EventType(org.zalando.nakadi.domain.EventType) Meter(com.codahale.metrics.Meter) Closeable(java.io.Closeable) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) IOException(java.io.IOException) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) EventConsumer(org.zalando.nakadi.repository.EventConsumer) Counter(com.codahale.metrics.Counter) EventStream(org.zalando.nakadi.service.EventStream) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) RequestMapping(org.springframework.web.bind.annotation.RequestMapping)

Aggregations

Counter (com.codahale.metrics.Counter)1 Meter (com.codahale.metrics.Meter)1 MetricRegistry (com.codahale.metrics.MetricRegistry)1 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 ImmutableList (com.google.common.collect.ImmutableList)1 Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Collectors (java.util.stream.Collectors)1 Nullable (javax.annotation.Nullable)1 HttpServletRequest (javax.servlet.http.HttpServletRequest)1 HttpServletResponse (javax.servlet.http.HttpServletResponse)1