Examples with NakadiCursor org.zalando.nakadi.domain.NakadiCursor used on opensource projects

Example 46 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorOperationsService method moveBack.

private NakadiCursor moveBack(final ShiftedNakadiCursor cursor) {
    NakadiCursor currentCursor = cursor.getNakadiCursor();
    long toMoveBack = -cursor.getShift();
    while (toMoveBack > 0) {
        final long totalBefore = numberOfEventsBeforeCursor(currentCursor);
        if (totalBefore < toMoveBack) {
            // +1 is because end is inclusive
            toMoveBack -= totalBefore + 1;
            // begin event that is not within limits)
            if (toMoveBack == 0) {
                toMoveBack += totalBefore + 1;
                break;
            }
            final Timeline prevTimeline = getTimeline(currentCursor.getEventType(), currentCursor.getTimeline().getOrder() - 1);
            // When moving back latest position is always defined
            currentCursor = prevTimeline.getLatestPosition().toNakadiCursor(prevTimeline, currentCursor.getPartition());
        } else {
            break;
        }
    }
    if (toMoveBack != 0) {
        currentCursor = currentCursor.shiftWithinTimeline(-toMoveBack);
    }
    return currentCursor;
}

Example 47 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class CursorsService method validateStreamId.

private void validateStreamId(final List<NakadiCursor> cursors, final String streamId, final ZkSubscriptionClient subscriptionClient) throws ServiceUnavailableException, InvalidCursorException, InvalidStreamIdException {
    if (!uuidGenerator.isUUID(streamId)) {
        throw new InvalidStreamIdException(String.format("Stream id has to be valid UUID, but `%s was provided", streamId), streamId);
    }
    if (!subscriptionClient.isActiveSession(streamId)) {
        throw new InvalidStreamIdException("Session with stream id " + streamId + " not found", streamId);
    }
    final Map<EventTypePartition, String> partitionSessions = Stream.of(subscriptionClient.getTopology().getPartitions()).collect(Collectors.toMap(Partition::getKey, Partition::getSession));
    for (final NakadiCursor cursor : cursors) {
        final EventTypePartition etPartition = cursor.getEventTypePartition();
        final String partitionSession = partitionSessions.get(etPartition);
        if (partitionSession == null) {
            throw new InvalidCursorException(CursorError.PARTITION_NOT_FOUND, cursor);
        }
        if (!streamId.equals(partitionSession)) {
            throw new InvalidStreamIdException("Cursor " + cursor + " cannot be committed with stream id " + streamId, streamId);
        }
    }
}

Example 48 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class EventStream method streamEvents.

public void streamEvents(final AtomicBoolean connectionReady, final Runnable checkAuthorization) {
    try {
        int messagesRead = 0;
        final Map<String, Integer> keepAliveInARow = createMapWithPartitionKeys(partition -> 0);
        final Map<String, List<byte[]>> currentBatches = createMapWithPartitionKeys(partition -> Lists.newArrayList());
        // Partition to NakadiCursor.
        final Map<String, NakadiCursor> latestOffsets = config.getCursors().stream().collect(Collectors.toMap(NakadiCursor::getPartition, c -> c));
        final long start = currentTimeMillis();
        final Map<String, Long> batchStartTimes = createMapWithPartitionKeys(partition -> start);
        final List<ConsumedEvent> consumedEvents = new LinkedList<>();
        long lastKpiEventSent = System.currentTimeMillis();
        long bytesInMemory = 0;
        while (connectionReady.get() && !blacklistService.isConsumptionBlocked(config.getEtName(), config.getConsumingClient().getClientId())) {
            checkAuthorization.run();
            if (consumedEvents.isEmpty()) {
                // TODO: There are a lot of optimizations here, one can significantly improve code by processing
                // all events at the same time, instead of processing one by one.
                consumedEvents.addAll(eventConsumer.readEvents());
            }
            final Optional<ConsumedEvent> eventOrEmpty = consumedEvents.isEmpty() ? Optional.empty() : Optional.of(consumedEvents.remove(0));
            if (eventOrEmpty.isPresent()) {
                final ConsumedEvent event = eventOrEmpty.get();
                // update offset for the partition of event that was read
                latestOffsets.put(event.getPosition().getPartition(), event.getPosition());
                // put message to batch
                currentBatches.get(event.getPosition().getPartition()).add(event.getEvent());
                messagesRead++;
                bytesInMemory += event.getEvent().length;
                // if we read the message - reset keep alive counter for this partition
                keepAliveInARow.put(event.getPosition().getPartition(), 0);
            }
            // for each partition check if it's time to send the batch
            for (final String partition : latestOffsets.keySet()) {
                final long timeSinceBatchStart = currentTimeMillis() - batchStartTimes.get(partition);
                if (config.getBatchTimeout() * 1000 <= timeSinceBatchStart || currentBatches.get(partition).size() >= config.getBatchLimit()) {
                    final List<byte[]> eventsToSend = currentBatches.get(partition);
                    sendBatch(latestOffsets.get(partition), eventsToSend);
                    if (!eventsToSend.isEmpty()) {
                        bytesInMemory -= eventsToSend.stream().mapToLong(v -> v.length).sum();
                        eventsToSend.clear();
                    } else {
                        // if we hit keep alive count limit - close the stream
                        keepAliveInARow.put(partition, keepAliveInARow.get(partition) + 1);
                    }
                    batchStartTimes.put(partition, currentTimeMillis());
                }
            }
            // Dump some data that is exceeding memory limits
            while (isMemoryLimitReached(bytesInMemory)) {
                final Map.Entry<String, List<byte[]>> heaviestPartition = currentBatches.entrySet().stream().max(Comparator.comparing(entry -> entry.getValue().stream().mapToLong(event -> event.length).sum())).get();
                sendBatch(latestOffsets.get(heaviestPartition.getKey()), heaviestPartition.getValue());
                final long freed = heaviestPartition.getValue().stream().mapToLong(v -> v.length).sum();
                LOG.warn("Memory limit reached for event type {}: {} bytes. Freed: {} bytes, {} messages", config.getEtName(), bytesInMemory, freed, heaviestPartition.getValue().size());
                bytesInMemory -= freed;
                // Init new batch for subscription
                heaviestPartition.getValue().clear();
                batchStartTimes.put(heaviestPartition.getKey(), currentTimeMillis());
            }
            if (lastKpiEventSent + kpiFrequencyMs < System.currentTimeMillis()) {
                final long count = kpiData.getAndResetNumberOfEventsSent();
                final long bytes = kpiData.getAndResetBytesSent();
                publishKpi(config.getConsumingClient(), count, bytes);
                lastKpiEventSent = System.currentTimeMillis();
            }
            // check if we reached keepAliveInARow for all the partitions; if yes - then close stream
            if (config.getStreamKeepAliveLimit() != 0) {
                final boolean keepAliveLimitReachedForAllPartitions = keepAliveInARow.values().stream().allMatch(keepAlives -> keepAlives >= config.getStreamKeepAliveLimit());
                if (keepAliveLimitReachedForAllPartitions) {
                    break;
                }
            }
            // check if we reached the stream timeout or message count limit
            final long timeSinceStart = currentTimeMillis() - start;
            if (config.getStreamTimeout() != 0 && timeSinceStart >= config.getStreamTimeout() * 1000 || config.getStreamLimit() != 0 && messagesRead >= config.getStreamLimit()) {
                for (final String partition : latestOffsets.keySet()) {
                    if (currentBatches.get(partition).size() > 0) {
                        sendBatch(latestOffsets.get(partition), currentBatches.get(partition));
                    }
                }
                break;
            }
        }
    } catch (final IOException e) {
        LOG.info("I/O error occurred when streaming events (possibly client closed connection)", e);
    } catch (final IllegalStateException e) {
        LOG.info("Error occurred when streaming events (possibly server closed connection)", e);
    } catch (final KafkaException e) {
        LOG.error("Error occurred when polling events from kafka; consumer: {}, event-type: {}", config.getConsumingClient().getClientId(), config.getEtName(), e);
    } finally {
        publishKpi(config.getConsumingClient(), kpiData.getAndResetNumberOfEventsSent(), kpiData.getAndResetBytesSent());
    }
}

Example 49 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class SubscriptionService method loadStats.

private List<SubscriptionEventTypeStats> loadStats(final Collection<EventType> eventTypes, final Optional<ZkSubscriptionNode> subscriptionNode, final ZkSubscriptionClient client, final List<PartitionEndStatistics> stats) throws ServiceTemporarilyUnavailableException, InconsistentStateException {
    final List<SubscriptionEventTypeStats> result = new ArrayList<>(eventTypes.size());
    final Collection<NakadiCursor> committedPositions = subscriptionNode.map(node -> loadCommittedPositions(node.getPartitions(), client)).orElse(Collections.emptyList());
    for (final EventType eventType : eventTypes) {
        final List<SubscriptionEventTypeStats.Partition> resultPartitions = new ArrayList<>(stats.size());
        for (final PartitionEndStatistics stat : stats) {
            final NakadiCursor lastPosition = stat.getLast();
            if (!lastPosition.getEventType().equals(eventType.getName())) {
                continue;
            }
            final Long distance = committedPositions.stream().filter(pos -> pos.getEventTypePartition().equals(lastPosition.getEventTypePartition())).findAny().map(committed -> {
                try {
                    return cursorOperationsService.calculateDistance(committed, lastPosition);
                } catch (final InvalidCursorOperation ex) {
                    throw new InconsistentStateException("Unexpected exception while calculating distance", ex);
                }
            }).orElse(null);
            final Partition.State state = subscriptionNode.map(node -> node.guessState(stat.getTimeline().getEventType(), stat.getPartition())).orElse(Partition.State.UNASSIGNED);
            final String streamId = subscriptionNode.map(node -> node.guessStream(stat.getTimeline().getEventType(), stat.getPartition())).orElse("");
            final SubscriptionEventTypeStats.Partition.AssignmentType assignmentType = subscriptionNode.map(node -> node.getPartitionAssignmentType(stat.getTimeline().getEventType(), stat.getPartition())).orElse(null);
            resultPartitions.add(new SubscriptionEventTypeStats.Partition(lastPosition.getPartition(), state.getDescription(), distance, streamId, assignmentType));
        }
        resultPartitions.sort(Comparator.comparing(SubscriptionEventTypeStats.Partition::getPartition));
        result.add(new SubscriptionEventTypeStats(eventType.getName(), resultPartitions));
    }
    return result;
}

Example 50 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class KafkaTopicRepository method convertToKafkaCursors.

private Map<NakadiCursor, KafkaCursor> convertToKafkaCursors(final List<NakadiCursor> cursors) throws ServiceUnavailableException, InvalidCursorException {
    final List<Timeline> timelines = cursors.stream().map(NakadiCursor::getTimeline).distinct().collect(toList());
    final List<PartitionStatistics> statistics = loadTopicStatistics(timelines);
    final Map<NakadiCursor, KafkaCursor> result = new HashMap<>();
    for (final NakadiCursor position : cursors) {
        validateCursorForNulls(position);
        final Optional<PartitionStatistics> partition = statistics.stream().filter(t -> Objects.equals(t.getPartition(), position.getPartition())).filter(t -> Objects.equals(t.getTimeline().getTopic(), position.getTopic())).findAny();
        if (!partition.isPresent()) {
            throw new InvalidCursorException(PARTITION_NOT_FOUND, position);
        }
        final KafkaCursor toCheck = position.asKafkaCursor();
        // Checking oldest position
        final KafkaCursor oldestCursor = KafkaCursor.fromNakadiCursor(partition.get().getBeforeFirst());
        if (toCheck.compareTo(oldestCursor) < 0) {
            throw new InvalidCursorException(UNAVAILABLE, position);
        }
        // checking newest position
        final KafkaCursor newestPosition = KafkaCursor.fromNakadiCursor(partition.get().getLast());
        if (toCheck.compareTo(newestPosition) > 0) {
            throw new InvalidCursorException(UNAVAILABLE, position);
        } else {
            result.put(position, toCheck);
        }
    }
    return result;
}