Examples with EventTypePartition org.zalando.nakadi.domain.EventTypePartition used on opensource projects

Example 11 with EventTypePartition

use of org.zalando.nakadi.domain.EventTypePartition in project nakadi by zalando.

the class StreamingState method reconfigureKafkaConsumer.

private void reconfigureKafkaConsumer(final boolean forceSeek) {
    if (eventConsumer == null) {
        throw new IllegalStateException("kafkaConsumer should not be null when calling reconfigureKafkaConsumer method");
    }
    final Set<EventTypePartition> newAssignment = offsets.keySet().stream().filter(o -> !this.releasingPartitions.containsKey(o)).collect(Collectors.toSet());
    if (forceSeek) {
        // removing all the current assignments for real consumer.
        try {
            eventConsumer.reassign(Collections.emptyList());
        } catch (final NakadiException | InvalidCursorException ex) {
            throw new NakadiRuntimeException(ex);
        }
    }
    final Set<EventTypePartition> currentAssignment = eventConsumer.getAssignment();
    getLog().info("Changing kafka assignment from {} to {}", Arrays.deepToString(currentAssignment.toArray()), Arrays.deepToString(newAssignment.toArray()));
    if (!currentAssignment.equals(newAssignment)) {
        try {
            final Map<EventTypePartition, NakadiCursor> beforeFirst = getBeforeFirstCursors(newAssignment);
            final List<NakadiCursor> cursors = newAssignment.stream().map(pk -> {
                final NakadiCursor beforeFirstAvailable = beforeFirst.get(pk);
                // Checks that current cursor is still available in storage
                offsets.get(pk).ensureDataAvailable(beforeFirstAvailable);
                return offsets.get(pk).getSentOffset();
            }).collect(Collectors.toList());
            eventConsumer.reassign(cursors);
        } catch (NakadiException | InvalidCursorException ex) {
            throw new NakadiRuntimeException(ex);
        }
    }
}

Example 12 with EventTypePartition

use of org.zalando.nakadi.domain.EventTypePartition in project nakadi by zalando.

the class AbstractZkSubscriptionClient method commitOffsets.

@Override
public List<Boolean> commitOffsets(final List<SubscriptionCursorWithoutToken> cursors, final Comparator<SubscriptionCursorWithoutToken> comparator) {
    final Map<EventTypePartition, List<SubscriptionCursorWithoutToken>> grouped = cursors.stream().collect(Collectors.groupingBy(SubscriptionCursorWithoutToken::getEventTypePartition));
    try {
        final Map<EventTypePartition, Iterator<Boolean>> committedOverall = new HashMap<>();
        for (final Map.Entry<EventTypePartition, List<SubscriptionCursorWithoutToken>> entry : grouped.entrySet()) {
            final String offsetPath = getOffsetPath(entry.getKey());
            final List<Boolean> committed;
            committed = executeWithRetry(() -> {
                final Stat stat = new Stat();
                final byte[] currentOffsetData = getCurator().getData().storingStatIn(stat).forPath(offsetPath);
                final String currentMaxOffset = new String(currentOffsetData, UTF_8);
                SubscriptionCursorWithoutToken currentMaxCursor = new SubscriptionCursorWithoutToken(entry.getKey().getEventType(), entry.getKey().getPartition(), currentMaxOffset);
                final List<Boolean> commits = Lists.newArrayList();
                for (final SubscriptionCursorWithoutToken cursor : entry.getValue()) {
                    if (comparator.compare(cursor, currentMaxCursor) > 0) {
                        currentMaxCursor = cursor;
                        commits.add(true);
                    } else {
                        commits.add(false);
                    }
                }
                if (!currentMaxCursor.getOffset().equals(currentMaxOffset)) {
                    getLog().info("Committing {} to {}", currentMaxCursor.getOffset(), offsetPath);
                    getCurator().setData().withVersion(stat.getVersion()).forPath(offsetPath, currentMaxCursor.getOffset().getBytes(Charsets.UTF_8));
                }
                return commits;
            }, new RetryForSpecifiedCountStrategy<List<Boolean>>(COMMIT_CONFLICT_RETRY_TIMES).withExceptionsThatForceRetry(KeeperException.BadVersionException.class));
            committedOverall.put(entry.getKey(), Optional.ofNullable(committed).orElse(Collections.nCopies(entry.getValue().size(), false)).iterator());
        }
        return cursors.stream().map(cursor -> committedOverall.get(cursor.getEventTypePartition()).next()).collect(Collectors.toList());
    } catch (final Exception ex) {
        throw new NakadiRuntimeException(ex);
    }
}

Example 13 with EventTypePartition

use of org.zalando.nakadi.domain.EventTypePartition in project nakadi by zalando.

the class NewZkSubscriptionClient method transfer.

@Override
public void transfer(final String sessionId, final Collection<EventTypePartition> partitions) throws NakadiRuntimeException, SubscriptionNotInitializedException {
    getLog().info("session " + sessionId + " releases partitions " + partitions);
    final Topology topology = getTopology();
    final List<Partition> changeSet = new ArrayList<>();
    for (final EventTypePartition etp : partitions) {
        final Partition candidate = Stream.of(topology.getPartitions()).filter(p -> p.getKey().equals(etp)).findAny().get();
        if (sessionId.equals(candidate.getSession()) && candidate.getState() == Partition.State.REASSIGNING) {
            changeSet.add(candidate.toState(Partition.State.ASSIGNED, candidate.getNextSession(), null));
        }
    }
    if (!changeSet.isEmpty()) {
        // The list of sessions didn't change, therefore one should not update sessionsHash.
        updatePartitionsConfiguration(topology.getSessionsHash(), changeSet.toArray(new Partition[changeSet.size()]));
    }
}

Example 14 with EventTypePartition

use of org.zalando.nakadi.domain.EventTypePartition in project nakadi by zalando.

the class CursorsService method getSubscriptionCursors.

public List<SubscriptionCursorWithoutToken> getSubscriptionCursors(final String subscriptionId) throws NakadiException, ServiceTemporarilyUnavailableException {
    final Subscription subscription = subscriptionRepository.getSubscription(subscriptionId);
    final ZkSubscriptionClient zkSubscriptionClient = zkSubscriptionFactory.createClient(subscription, "subscription." + subscriptionId + ".get_cursors");
    final ImmutableList.Builder<SubscriptionCursorWithoutToken> cursorsListBuilder = ImmutableList.builder();
    Partition[] partitions;
    try {
        partitions = zkSubscriptionClient.getTopology().getPartitions();
    } catch (final SubscriptionNotInitializedException ex) {
        partitions = new Partition[] {};
    }
    final Map<EventTypePartition, SubscriptionCursorWithoutToken> positions = zkSubscriptionClient.getOffsets(Stream.of(partitions).map(Partition::getKey).collect(Collectors.toList()));
    for (final Partition p : partitions) {
        cursorsListBuilder.add(positions.get(p.getKey()));
    }
    return cursorsListBuilder.build();
}

Example 15 with EventTypePartition

use of org.zalando.nakadi.domain.EventTypePartition in project nakadi by zalando.

the class SubscriptionRebalancer method apply.

@Override
public Partition[] apply(final Collection<Session> sessions, final Partition[] currentPartitions) {
    final List<String> activeSessions = sessions.stream().map(Session::getId).collect(Collectors.toList());
    final List<Partition> partitionsLeft = Lists.newArrayList(currentPartitions);
    final List<Partition> changedPartitions = new ArrayList<>();
    final List<Session> sessionsWithSpecifiedPartitions = sessions.stream().filter(s -> !s.getRequestedPartitions().isEmpty()).collect(Collectors.toList());
    // go through all sessions that directly requested partitions to stream
    for (final Session session : sessionsWithSpecifiedPartitions) {
        for (final EventTypePartition requestedPartition : session.getRequestedPartitions()) {
            // find a partition that is requested and assign it to a session that requests it
            final Partition partition = partitionsLeft.stream().filter(p -> p.getKey().equals(requestedPartition)).findFirst().orElseThrow(() -> new RebalanceConflictException("Two existing sessions request the same partition: " + requestedPartition));
            partitionsLeft.remove(partition);
            // if this partition is not assigned to this session - move it
            if (!session.getId().equals(partition.getSession())) {
                final Partition movedPartition = partition.moveToSessionId(session.getId(), activeSessions);
                changedPartitions.add(movedPartition);
            }
        }
    }
    // for the rest of partitions/sessions perform a rebalance based on partitions count
    final List<Session> autoBalanceSessions = sessions.stream().filter(s -> s.getRequestedPartitions().isEmpty()).collect(Collectors.toList());
    if (!autoBalanceSessions.isEmpty() && !partitionsLeft.isEmpty()) {
        final Partition[] partitionsChangedByAutoRebalance = rebalanceByWeight(autoBalanceSessions, partitionsLeft.toArray(new Partition[partitionsLeft.size()]));
        changedPartitions.addAll(Arrays.asList(partitionsChangedByAutoRebalance));
    }
    return changedPartitions.toArray(new Partition[changedPartitions.size()]);
}