Examples with Partition org.zalando.nakadi.service.subscription.model.Partition used on opensource projects

Example 11 with Partition

use of org.zalando.nakadi.service.subscription.model.Partition in project nakadi by zalando.

the class NewZkSubscriptionClient method transfer.

@Override
public void transfer(final String sessionId, final Collection<EventTypePartition> partitions) throws NakadiRuntimeException, SubscriptionNotInitializedException {
    getLog().info("session " + sessionId + " releases partitions " + partitions);
    final Topology topology = getTopology();
    final List<Partition> changeSet = new ArrayList<>();
    for (final EventTypePartition etp : partitions) {
        final Partition candidate = Stream.of(topology.getPartitions()).filter(p -> p.getKey().equals(etp)).findAny().get();
        if (sessionId.equals(candidate.getSession()) && candidate.getState() == Partition.State.REASSIGNING) {
            changeSet.add(candidate.toState(Partition.State.ASSIGNED, candidate.getNextSession(), null));
        }
    }
    if (!changeSet.isEmpty()) {
        // The list of sessions didn't change, therefore one should not update sessionsHash.
        updatePartitionsConfiguration(topology.getSessionsHash(), changeSet.toArray(new Partition[changeSet.size()]));
    }
}

Example 12 with Partition

use of org.zalando.nakadi.service.subscription.model.Partition in project nakadi by zalando.

the class CursorsService method getSubscriptionCursors.

public List<SubscriptionCursorWithoutToken> getSubscriptionCursors(final String subscriptionId) throws NakadiException, ServiceTemporarilyUnavailableException {
    final Subscription subscription = subscriptionRepository.getSubscription(subscriptionId);
    final ZkSubscriptionClient zkSubscriptionClient = zkSubscriptionFactory.createClient(subscription, "subscription." + subscriptionId + ".get_cursors");
    final ImmutableList.Builder<SubscriptionCursorWithoutToken> cursorsListBuilder = ImmutableList.builder();
    Partition[] partitions;
    try {
        partitions = zkSubscriptionClient.getTopology().getPartitions();
    } catch (final SubscriptionNotInitializedException ex) {
        partitions = new Partition[] {};
    }
    final Map<EventTypePartition, SubscriptionCursorWithoutToken> positions = zkSubscriptionClient.getOffsets(Stream.of(partitions).map(Partition::getKey).collect(Collectors.toList()));
    for (final Partition p : partitions) {
        cursorsListBuilder.add(positions.get(p.getKey()));
    }
    return cursorsListBuilder.build();
}

Example 13 with Partition

use of org.zalando.nakadi.service.subscription.model.Partition in project nakadi by zalando.

the class SubscriptionRebalancer method apply.

@Override
public Partition[] apply(final Collection<Session> sessions, final Partition[] currentPartitions) {
    final List<String> activeSessions = sessions.stream().map(Session::getId).collect(Collectors.toList());
    final List<Partition> partitionsLeft = Lists.newArrayList(currentPartitions);
    final List<Partition> changedPartitions = new ArrayList<>();
    final List<Session> sessionsWithSpecifiedPartitions = sessions.stream().filter(s -> !s.getRequestedPartitions().isEmpty()).collect(Collectors.toList());
    // go through all sessions that directly requested partitions to stream
    for (final Session session : sessionsWithSpecifiedPartitions) {
        for (final EventTypePartition requestedPartition : session.getRequestedPartitions()) {
            // find a partition that is requested and assign it to a session that requests it
            final Partition partition = partitionsLeft.stream().filter(p -> p.getKey().equals(requestedPartition)).findFirst().orElseThrow(() -> new RebalanceConflictException("Two existing sessions request the same partition: " + requestedPartition));
            partitionsLeft.remove(partition);
            // if this partition is not assigned to this session - move it
            if (!session.getId().equals(partition.getSession())) {
                final Partition movedPartition = partition.moveToSessionId(session.getId(), activeSessions);
                changedPartitions.add(movedPartition);
            }
        }
    }
    // for the rest of partitions/sessions perform a rebalance based on partitions count
    final List<Session> autoBalanceSessions = sessions.stream().filter(s -> s.getRequestedPartitions().isEmpty()).collect(Collectors.toList());
    if (!autoBalanceSessions.isEmpty() && !partitionsLeft.isEmpty()) {
        final Partition[] partitionsChangedByAutoRebalance = rebalanceByWeight(autoBalanceSessions, partitionsLeft.toArray(new Partition[partitionsLeft.size()]));
        changedPartitions.addAll(Arrays.asList(partitionsChangedByAutoRebalance));
    }
    return changedPartitions.toArray(new Partition[changedPartitions.size()]);
}

Example 14 with Partition

use of org.zalando.nakadi.service.subscription.model.Partition in project nakadi by zalando.

the class CursorsServiceAT method whenMultipleCursorsForSamePartitionThenResultsAreCorrect.

@Test
public void whenMultipleCursorsForSamePartitionThenResultsAreCorrect() throws Exception {
    setPartitions(new Partition[] { new Partition(etName, P1, streamId, null, Partition.State.ASSIGNED), new Partition(etName, P2, streamId, null, Partition.State.ASSIGNED) });
    CURATOR.setData().forPath(offsetPath(P1), "000000000000000100".getBytes(UTF_8));
    CURATOR.setData().forPath(offsetPath(P2), "000000000000000800".getBytes(UTF_8));
    registerNakadiCursor(NakadiCursor.of(timeline, P1, "000000000000000100"));
    registerNakadiCursor(NakadiCursor.of(timeline, P2, "000000000000000800"));
    testCursors = ImmutableList.of(NakadiCursor.of(timeline, P1, "000000000000000105"), NakadiCursor.of(timeline, P1, "000000000000000106"), NakadiCursor.of(timeline, P1, "000000000000000102"), NakadiCursor.of(timeline, P1, "000000000000000096"), NakadiCursor.of(timeline, P1, "000000000000000130"), NakadiCursor.of(timeline, P2, "000000000000000800"), NakadiCursor.of(timeline, P2, "000000000000000820"), NakadiCursor.of(timeline, P1, "000000000000000120"), NakadiCursor.of(timeline, P1, "000000000000000121"), NakadiCursor.of(timeline, P2, "000000000000000825"));
    testCursors.forEach(this::registerNakadiCursor);
    final List<Boolean> commitResult = cursorsService.commitCursors(streamId, sid, testCursors);
    assertThat(commitResult, equalTo(ImmutableList.of(true, true, false, false, true, false, true, false, false, true)));
    checkCurrentOffsetInZk(P1, "000000000000000130");
    checkCurrentOffsetInZk(P2, "000000000000000825");
}

Example 15 with Partition

use of org.zalando.nakadi.service.subscription.model.Partition in project nakadi by zalando.

the class CursorsServiceAT method whenGetSubscriptionCursorsThenOk.

@Test
public void whenGetSubscriptionCursorsThenOk() throws Exception {
    setPartitions(new Partition[] { new Partition(etName, P1, "", "", Partition.State.UNASSIGNED), new Partition(etName, P2, "", "", Partition.State.UNASSIGNED) });
    final List<SubscriptionCursorWithoutToken> cursors = cursorsService.getSubscriptionCursors(sid);
    assertThat(ImmutableSet.copyOf(cursors), equalTo(ImmutableSet.of(new SubscriptionCursorWithoutToken(etName, P1, OLD_OFFSET), new SubscriptionCursorWithoutToken(etName, P2, OLD_OFFSET))));
}