Examples with NakadiCursor org.zalando.nakadi.domain.NakadiCursor used on opensource projects

Example 51 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class SubscriptionValidationService method validateInitialCursors.

private void validateInitialCursors(final SubscriptionBase subscription, final List<EventTypePartition> allPartitions) throws WrongInitialCursorsException, RepositoryProblemException {
    final boolean cursorsMissing = allPartitions.stream().anyMatch(p -> !subscription.getInitialCursors().stream().anyMatch(p::ownsCursor));
    if (cursorsMissing) {
        throw new WrongInitialCursorsException("initial_cursors should contain cursors for all partitions of subscription");
    }
    final boolean hasCursorForWrongPartition = subscription.getInitialCursors().stream().anyMatch(c -> !allPartitions.contains(new EventTypePartition(c.getEventType(), c.getPartition())));
    if (hasCursorForWrongPartition) {
        throw new WrongInitialCursorsException("initial_cursors should contain cursors only for partitions of this subscription");
    }
    if (subscription.getInitialCursors().size() > allPartitions.size()) {
        throw new WrongInitialCursorsException("there should be no more than 1 cursor for each partition in initial_cursors");
    }
    try {
        for (final SubscriptionCursorWithoutToken cursor : subscription.getInitialCursors()) {
            final NakadiCursor nakadiCursor = cursorConverter.convert(cursor);
            if (nakadiCursor.getTimeline().isDeleted()) {
                throw new InvalidCursorException(UNAVAILABLE, nakadiCursor);
            }
            timelineService.getTopicRepository(nakadiCursor.getTimeline()).validateReadCursors(Collections.singletonList(nakadiCursor));
        }
    } catch (final InvalidCursorException ex) {
        throw new WrongInitialCursorsException(ex.getMessage(), ex);
    } catch (final NakadiException ex) {
        throw new RepositoryProblemException("Topic repository problem occurred when validating cursors", ex);
    }
}

Example 52 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class PartitionData method onCommitOffset.

CommitResult onCommitOffset(final NakadiCursor offset) {
    boolean seekKafka = false;
    if (comparator.compare(offset, sentOffset) > 0) {
        log.error("Commit in future: current: {}, committed {} will skip sending obsolete data", sentOffset, commitOffset);
        seekKafka = true;
        sentOffset = offset;
    }
    final long committed;
    if (comparator.compare(offset, commitOffset) >= 0) {
        final Set<NakadiCursor> committedCursors = allCursorsOrdered.headSet(offset, true);
        committed = committedCursors.size();
        commitOffset = offset;
        // Operation is cascaded to allCursorsOrdered set.
        committedCursors.clear();
    } else {
        log.error("Commits in past are evil!: Committing in {} while current commit is {}", offset, commitOffset);
        // Commit in past occurred. One should move storage pointer to sentOffset.
        seekKafka = true;
        commitOffset = offset;
        sentOffset = commitOffset;
        allCursorsOrdered.clear();
        nakadiEvents.clear();
        bytesInMemory = 0L;
        committed = 0;
    }
    while (!nakadiEvents.isEmpty() && comparator.compare(nakadiEvents.get(0).getPosition(), commitOffset) <= 0) {
        final ConsumedEvent evt = nakadiEvents.remove(0);
        bytesInMemory -= evt.getEvent().length;
    }
    return new CommitResult(seekKafka, committed);
}

Example 53 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class StreamingState method flushData.

private void flushData(final EventTypePartition pk, final List<ConsumedEvent> data, final Optional<String> metadata) {
    try {
        final NakadiCursor sentOffset = offsets.get(pk).getSentOffset();
        final SubscriptionCursor cursor = getContext().getCursorConverter().convert(sentOffset, getContext().getCursorTokenService().generateToken());
        final int batchSize = getContext().getWriter().writeSubscriptionBatch(getOut().getOutputStream(), cursor, data, metadata);
        bytesSentMeterPerSubscription.mark(batchSize);
        final StreamKpiData kpiData = kpiDataPerEventType.get(pk.getEventType());
        kpiData.addBytesSent(batchSize);
        kpiData.addNumberOfEventsSent(data.size());
        batchesSent++;
    } catch (final IOException e) {
        getLog().error("Failed to write data to output.", e);
        shutdownGracefully("Failed to write data to output");
    }
}

Example 54 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class StreamingState method offsetChanged.

void offsetChanged(final EventTypePartition key) {
    if (offsets.containsKey(key)) {
        final PartitionData data = offsets.get(key);
        final NakadiCursor cursor = createNakadiCursor(data.getSubscription().getData());
        final PartitionData.CommitResult commitResult = data.onCommitOffset(cursor);
        if (commitResult.seekOnKafka) {
            reconfigureKafkaConsumer(true);
        }
        if (commitResult.committedCount > 0) {
            committedEvents += commitResult.committedCount;
            this.lastCommitMillis = System.currentTimeMillis();
            streamToOutput();
        }
        if (getParameters().isStreamLimitReached(committedEvents)) {
            final String debugMessage = "Stream limit in events reached: " + committedEvents;
            sendMetadata(debugMessage);
            shutdownGracefully(debugMessage);
        }
        if (releasingPartitions.containsKey(key) && data.isCommitted()) {
            reassignCommitted();
            logPartitionAssignment("New offset received for releasing partition " + key);
        }
    }
}

Example 55 with NakadiCursor

use of org.zalando.nakadi.domain.NakadiCursor in project nakadi by zalando.

the class StreamingState method addToStreaming.

private void addToStreaming(final Partition partition, final Map<EventTypePartition, SubscriptionCursorWithoutToken> cursorMap) {
    final NakadiCursor cursor = createNakadiCursor(cursorMap.get(partition.getKey()));
    getLog().info("Adding to streaming {} with start position {}", partition.getKey(), cursor);
    final ZkSubscription<SubscriptionCursorWithoutToken> subscription = getZk().subscribeForOffsetChanges(partition.getKey(), () -> addTask(() -> offsetChanged(partition.getKey())));
    final PartitionData pd = new PartitionData(getComparator(), subscription, cursor, LoggerFactory.getLogger("subscription." + getSessionId() + "." + partition.getKey()), System.currentTimeMillis());
    offsets.put(partition.getKey(), pd);
}