Search in sources :

Example 1 with NoConnectionSlotsException

use of org.zalando.nakadi.exceptions.NoConnectionSlotsException in project nakadi by zalando.

the class EventStreamController method streamEvents.

@RequestMapping(value = "/event-types/{name}/events", method = RequestMethod.GET)
public StreamingResponseBody streamEvents(@PathVariable("name") final String eventTypeName, @Nullable @RequestParam(value = "batch_limit", required = false) final Integer batchLimit, @Nullable @RequestParam(value = "stream_limit", required = false) final Integer streamLimit, @Nullable @RequestParam(value = "batch_flush_timeout", required = false) final Integer batchTimeout, @Nullable @RequestParam(value = "stream_timeout", required = false) final Integer streamTimeout, @Nullable @RequestParam(value = "stream_keep_alive_limit", required = false) final Integer streamKeepAliveLimit, @Nullable @RequestHeader(name = "X-nakadi-cursors", required = false) final String cursorsStr, final HttpServletRequest request, final HttpServletResponse response, final Client client) {
    final String flowId = FlowIdUtils.peek();
    return outputStream -> {
        FlowIdUtils.push(flowId);
        if (blacklistService.isConsumptionBlocked(eventTypeName, client.getClientId())) {
            writeProblemResponse(response, outputStream, Problem.valueOf(Response.Status.FORBIDDEN, "Application or event type is blocked"));
            return;
        }
        final AtomicBoolean connectionReady = closedConnectionsCrutch.listenForConnectionClose(request);
        Counter consumerCounter = null;
        EventStream eventStream = null;
        List<ConnectionSlot> connectionSlots = ImmutableList.of();
        final AtomicBoolean needCheckAuthorization = new AtomicBoolean(false);
        LOG.info("[X-NAKADI-CURSORS] \"{}\" {}", eventTypeName, Optional.ofNullable(cursorsStr).orElse("-"));
        try (Closeable ignore = eventTypeChangeListener.registerListener(et -> needCheckAuthorization.set(true), Collections.singletonList(eventTypeName))) {
            final EventType eventType = eventTypeRepository.findByName(eventTypeName);
            authorizeStreamRead(eventTypeName);
            // validate parameters
            final EventStreamConfig streamConfig = EventStreamConfig.builder().withBatchLimit(batchLimit).withStreamLimit(streamLimit).withBatchTimeout(batchTimeout).withStreamTimeout(streamTimeout).withStreamKeepAliveLimit(streamKeepAliveLimit).withEtName(eventTypeName).withConsumingClient(client).withCursors(getStreamingStart(eventType, cursorsStr)).withMaxMemoryUsageBytes(maxMemoryUsageBytes).build();
            // acquire connection slots to limit the number of simultaneous connections from one client
            if (featureToggleService.isFeatureEnabled(LIMIT_CONSUMERS_NUMBER)) {
                final List<String> partitions = streamConfig.getCursors().stream().map(NakadiCursor::getPartition).collect(Collectors.toList());
                connectionSlots = consumerLimitingService.acquireConnectionSlots(client.getClientId(), eventTypeName, partitions);
            }
            consumerCounter = metricRegistry.counter(metricNameFor(eventTypeName, CONSUMERS_COUNT_METRIC_NAME));
            consumerCounter.inc();
            final String kafkaQuotaClientId = getKafkaQuotaClientId(eventTypeName, client);
            response.setStatus(HttpStatus.OK.value());
            response.setHeader("Warning", "299 - nakadi - the Low-level API is deprecated and will " + "be removed from a future release. Please consider migrating to the Subscriptions API.");
            response.setContentType("application/x-json-stream");
            final EventConsumer eventConsumer = timelineService.createEventConsumer(kafkaQuotaClientId, streamConfig.getCursors());
            final String bytesFlushedMetricName = MetricUtils.metricNameForLoLAStream(client.getClientId(), eventTypeName);
            final Meter bytesFlushedMeter = this.streamMetrics.meter(bytesFlushedMetricName);
            eventStream = eventStreamFactory.createEventStream(outputStream, eventConsumer, streamConfig, bytesFlushedMeter);
            // Flush status code to client
            outputStream.flush();
            eventStream.streamEvents(connectionReady, () -> {
                if (needCheckAuthorization.getAndSet(false)) {
                    authorizeStreamRead(eventTypeName);
                }
            });
        } catch (final UnparseableCursorException e) {
            LOG.debug("Incorrect syntax of X-nakadi-cursors header: {}. Respond with BAD_REQUEST.", e.getCursors(), e);
            writeProblemResponse(response, outputStream, BAD_REQUEST, e.getMessage());
        } catch (final NoSuchEventTypeException e) {
            writeProblemResponse(response, outputStream, NOT_FOUND, "topic not found");
        } catch (final NoConnectionSlotsException e) {
            LOG.debug("Connection creation failed due to exceeding max connection count");
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final NakadiException e) {
            LOG.error("Error while trying to stream events.", e);
            writeProblemResponse(response, outputStream, e.asProblem());
        } catch (final InvalidCursorException e) {
            writeProblemResponse(response, outputStream, PRECONDITION_FAILED, e.getMessage());
        } catch (final AccessDeniedException e) {
            writeProblemResponse(response, outputStream, FORBIDDEN, e.explain());
        } catch (final Exception e) {
            LOG.error("Error while trying to stream events. Respond with INTERNAL_SERVER_ERROR.", e);
            writeProblemResponse(response, outputStream, INTERNAL_SERVER_ERROR, e.getMessage());
        } finally {
            connectionReady.set(false);
            consumerLimitingService.releaseConnectionSlots(connectionSlots);
            if (consumerCounter != null) {
                consumerCounter.dec();
            }
            if (eventStream != null) {
                eventStream.close();
            }
            try {
                outputStream.flush();
            } finally {
                outputStream.close();
            }
        }
    };
}
Also used : PathVariable(org.springframework.web.bind.annotation.PathVariable) RequestParam(org.springframework.web.bind.annotation.RequestParam) FlowIdUtils(org.zalando.nakadi.util.FlowIdUtils) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) Problem(org.zalando.problem.Problem) Map(java.util.Map) Counter(com.codahale.metrics.Counter) TimelineService(org.zalando.nakadi.service.timeline.TimelineService) TypeReference(com.fasterxml.jackson.core.type.TypeReference) FeatureToggleService(org.zalando.nakadi.service.FeatureToggleService) EventTypeChangeListener(org.zalando.nakadi.service.EventTypeChangeListener) BAD_REQUEST(javax.ws.rs.core.Response.Status.BAD_REQUEST) ConsumerLimitingService(org.zalando.nakadi.service.ConsumerLimitingService) TopicRepository(org.zalando.nakadi.repository.TopicRepository) ConnectionSlot(org.zalando.nakadi.service.ConnectionSlot) NOT_FOUND(javax.ws.rs.core.Response.Status.NOT_FOUND) PartitionStatistics(org.zalando.nakadi.domain.PartitionStatistics) RequestMethod(org.springframework.web.bind.annotation.RequestMethod) StreamingResponseBody(org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) BlacklistService(org.zalando.nakadi.service.BlacklistService) RestController(org.springframework.web.bind.annotation.RestController) Collectors(java.util.stream.Collectors) List(java.util.List) Response(javax.ws.rs.core.Response) Timeline(org.zalando.nakadi.domain.Timeline) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) Optional(java.util.Optional) RequestHeader(org.springframework.web.bind.annotation.RequestHeader) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) Client(org.zalando.nakadi.security.Client) Storage(org.zalando.nakadi.domain.Storage) ClosedConnectionsCrutch(org.zalando.nakadi.service.ClosedConnectionsCrutch) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) RequestMapping(org.springframework.web.bind.annotation.RequestMapping) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Cursor(org.zalando.nakadi.view.Cursor) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) Meter(com.codahale.metrics.Meter) HttpServletRequest(javax.servlet.http.HttpServletRequest) ImmutableList(com.google.common.collect.ImmutableList) EventStreamFactory(org.zalando.nakadi.service.EventStreamFactory) Qualifier(org.springframework.beans.factory.annotation.Qualifier) PRECONDITION_FAILED(javax.ws.rs.core.Response.Status.PRECONDITION_FAILED) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) EventType(org.zalando.nakadi.domain.EventType) MetricRegistry(com.codahale.metrics.MetricRegistry) Logger(org.slf4j.Logger) INTERNAL_SERVER_ERROR(javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) HttpServletResponse(javax.servlet.http.HttpServletResponse) MetricUtils.metricNameFor(org.zalando.nakadi.metrics.MetricUtils.metricNameFor) IOException(java.io.IOException) FORBIDDEN(javax.ws.rs.core.Response.Status.FORBIDDEN) EventStream(org.zalando.nakadi.service.EventStream) AuthorizationValidator(org.zalando.nakadi.service.AuthorizationValidator) HttpStatus(org.springframework.http.HttpStatus) EventConsumer(org.zalando.nakadi.repository.EventConsumer) EventTypeRepository(org.zalando.nakadi.repository.EventTypeRepository) Closeable(java.io.Closeable) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) LIMIT_CONSUMERS_NUMBER(org.zalando.nakadi.service.FeatureToggleService.Feature.LIMIT_CONSUMERS_NUMBER) CursorError(org.zalando.nakadi.domain.CursorError) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MetricUtils(org.zalando.nakadi.metrics.MetricUtils) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) Collections(java.util.Collections) CursorConverter(org.zalando.nakadi.service.CursorConverter) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) NakadiCursor(org.zalando.nakadi.domain.NakadiCursor) EventType(org.zalando.nakadi.domain.EventType) Meter(com.codahale.metrics.Meter) Closeable(java.io.Closeable) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) InvalidCursorException(org.zalando.nakadi.exceptions.InvalidCursorException) ServiceTemporarilyUnavailableException(org.zalando.nakadi.exceptions.runtime.ServiceTemporarilyUnavailableException) UnparseableCursorException(org.zalando.nakadi.exceptions.UnparseableCursorException) AccessDeniedException(org.zalando.nakadi.exceptions.runtime.AccessDeniedException) IOException(java.io.IOException) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) NakadiException(org.zalando.nakadi.exceptions.NakadiException) InternalNakadiException(org.zalando.nakadi.exceptions.InternalNakadiException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) EventStreamConfig(org.zalando.nakadi.service.EventStreamConfig) EventConsumer(org.zalando.nakadi.repository.EventConsumer) Counter(com.codahale.metrics.Counter) EventStream(org.zalando.nakadi.service.EventStream) List(java.util.List) ArrayList(java.util.ArrayList) ImmutableList(com.google.common.collect.ImmutableList) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) NoSuchEventTypeException(org.zalando.nakadi.exceptions.NoSuchEventTypeException) RequestMapping(org.springframework.web.bind.annotation.RequestMapping)

Example 2 with NoConnectionSlotsException

use of org.zalando.nakadi.exceptions.NoConnectionSlotsException in project nakadi by zalando.

the class ConsumerLimitingService method acquireConnectionSlots.

@SuppressWarnings("unchecked")
public List<ConnectionSlot> acquireConnectionSlots(final String client, final String eventType, final List<String> partitions) throws NoConnectionSlotsException, ServiceUnavailableException {
    final List<String> partitionsWithNoFreeSlots = getPartitionsWithNoFreeSlots(client, eventType, partitions);
    if (partitionsWithNoFreeSlots.size() == 0) {
        final List<ConnectionSlot> slots = new ArrayList<>();
        final String lockZkPath = ZKPaths.makePath(LOCKS_ZK_PATH, client + "|" + eventType);
        try {
            return runLocked(() -> {
                // we need to check it again when we are under lock
                final List<String> occupiedPartitions = getPartitionsWithNoFreeSlots(client, eventType, partitions);
                if (occupiedPartitions.size() > 0) {
                    throw generateNoConnectionSlotsException(eventType, occupiedPartitions, client);
                }
                for (final String partition : partitions) {
                    final ConnectionSlot connectionSlot = acquireConnectionSlot(client, eventType, partition);
                    slots.add(connectionSlot);
                }
                return slots;
            }, zkLockFactory.createLock(lockZkPath));
        } catch (final NoConnectionSlotsException e) {
            throw e;
        } catch (final Exception e) {
            // in a case of failure release slots for partitions that already acquired slots
            slots.forEach(this::releaseConnectionSlot);
            throw new ServiceUnavailableException("Error communicating with zookeeper", e);
        }
    } else {
        throw generateNoConnectionSlotsException(eventType, partitionsWithNoFreeSlots, client);
    }
}
Also used : ArrayList(java.util.ArrayList) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) ServiceUnavailableException(org.zalando.nakadi.exceptions.ServiceUnavailableException) KeeperException(org.apache.zookeeper.KeeperException) IOException(java.io.IOException) NakadiRuntimeException(org.zalando.nakadi.exceptions.NakadiRuntimeException) NoConnectionSlotsException(org.zalando.nakadi.exceptions.NoConnectionSlotsException)

Aggregations

IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Counter (com.codahale.metrics.Counter)1 Meter (com.codahale.metrics.Meter)1 MetricRegistry (com.codahale.metrics.MetricRegistry)1 TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 ImmutableList (com.google.common.collect.ImmutableList)1 Closeable (java.io.Closeable)1 OutputStream (java.io.OutputStream)1 Collections (java.util.Collections)1 List (java.util.List)1 Map (java.util.Map)1 Optional (java.util.Optional)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Collectors (java.util.stream.Collectors)1 Nullable (javax.annotation.Nullable)1 HttpServletRequest (javax.servlet.http.HttpServletRequest)1 HttpServletResponse (javax.servlet.http.HttpServletResponse)1