Examples with SubscriptionCursor org.zalando.nakadi.view.SubscriptionCursor used on opensource projects

Example 11 with SubscriptionCursor

use of org.zalando.nakadi.view.SubscriptionCursor in project nakadi by zalando.

the class HilaAT method whenPatchThenCursorsAreInitializedAndPatched.

@Test(timeout = 15000)
public void whenPatchThenCursorsAreInitializedAndPatched() throws Exception {
    final EventType et = createEventType();
    publishEvents(et.getName(), 10, i -> "{\"foo\": \"bar\"}");
    final List<EventTypePartitionView> etStats = MAPPER.readValue(given().get("/event-types/{et}/partitions", et.getName()).getBody().asString(), new TypeReference<List<EventTypePartitionView>>() {
    });
    final EventTypePartitionView begin = etStats.get(0);
    final Subscription s = createSubscription(RandomSubscriptionBuilder.builder().withEventType(et.getName()).withStartFrom(END).buildSubscriptionBase());
    given().body(MAPPER.writeValueAsString(new ItemsWrapper<>(Collections.singletonList(new SubscriptionCursorWithoutToken(et.getName(), begin.getPartitionId(), begin.getOldestAvailableOffset()))))).contentType(JSON).patch("/subscriptions/{id}/cursors", s.getId()).then().statusCode(SC_NO_CONTENT);
    final ItemsWrapper<SubscriptionCursor> subscriptionCursors = MAPPER.readValue(given().get("/subscriptions/{id}/cursors", s.getId()).getBody().asString(), new TypeReference<ItemsWrapper<SubscriptionCursor>>() {
    });
    Assert.assertEquals(subscriptionCursors.getItems().size(), etStats.size());
    subscriptionCursors.getItems().forEach(item -> {
        if (item.getPartition().equals(begin.getPartitionId())) {
            Assert.assertEquals(begin.getOldestAvailableOffset(), item.getOffset());
        } else {
            Assert.assertEquals(begin.getNewestAvailableOffset(), item.getOffset());
        }
    });
}

Example 12 with SubscriptionCursor

use of org.zalando.nakadi.view.SubscriptionCursor in project nakadi by zalando.

the class HilaAT method whenStreamTimeoutReachedPossibleToCommit.

@Test(timeout = 10000)
public void whenStreamTimeoutReachedPossibleToCommit() throws Exception {
    final TestStreamingClient client = TestStreamingClient.create(URL, subscription.getId(), "batch_limit=1&stream_limit=2&stream_timeout=1").start();
    waitFor(() -> assertThat(client.getSessionId(), not(equalTo(SESSION_ID_UNKNOWN))));
    publishEvent(eventType.getName(), "{\"foo\":\"bar\"}");
    waitFor(() -> Assert.assertFalse(client.getBatches().isEmpty()), TimeUnit.SECONDS.toMillis(2), 100);
    final SubscriptionCursor toCommit = client.getBatches().get(0).getCursor();
    // connection is closed, and stream as well
    client.close();
    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
    final int statusCode = commitCursors(subscription.getId(), Collections.singletonList(toCommit), client.getSessionId());
    Assert.assertEquals(SC_NO_CONTENT, statusCode);
}

Example 13 with SubscriptionCursor

use of org.zalando.nakadi.view.SubscriptionCursor in project nakadi by zalando.

the class HilaAT method whenNoEventsThenBeginOffsetIsUsed.

@Test(timeout = 5000)
public void whenNoEventsThenBeginOffsetIsUsed() throws Exception {
    final TestStreamingClient client = TestStreamingClient.create(subscription.getId()).start();
    waitFor(() -> assertThat(client.getSessionId(), not(equalTo(SESSION_ID_UNKNOWN))));
    when().get("/subscriptions/{sid}/cursors", subscription.getId()).then().body("items[0].offset", equalTo("001-0001--1"));
    final int commitResult = commitCursors(subscription.getId(), ImmutableList.of(new SubscriptionCursor("0", Cursor.BEFORE_OLDEST_OFFSET, eventType.getName(), "abc")), client.getSessionId());
    assertThat(commitResult, equalTo(SC_OK));
}

Example 14 with SubscriptionCursor

use of org.zalando.nakadi.view.SubscriptionCursor in project nakadi by zalando.

the class TestStreamingClient method startWithAutocommit.

public TestStreamingClient startWithAutocommit(final Consumer<List<StreamBatch>> batchesListener) throws InterruptedException {
    this.batchesListener = batchesListener;
    final TestStreamingClient client = startInternal(true);
    final Thread autocommitThread = new Thread(() -> {
        int oldIdx = 0;
        while (client.isRunning()) {
            while (oldIdx < client.getBatches().size()) {
                final StreamBatch batch = client.getBatches().get(oldIdx);
                if (batch.getEvents() != null && !batch.getEvents().isEmpty()) {
                    try {
                        final SubscriptionCursor cursor = batch.getCursor();
                        final int responseCode = NakadiTestUtils.commitCursors(client.subscriptionId, Collections.singletonList(batch.getCursor()), client.getSessionId());
                        LOG.info("Committing " + responseCode + ": " + cursor);
                    } catch (JsonProcessingException e) {
                        throw new RuntimeException(e);
                    }
                }
                oldIdx += 1;
            }
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
    });
    autocommitThread.setDaemon(true);
    autocommitThread.start();
    return client;
}

Example 15 with SubscriptionCursor

use of org.zalando.nakadi.view.SubscriptionCursor in project nakadi by zalando.

the class HilaRebalanceAT method whenRebalanceThenPartitionsAreEquallyDistributedAndCommittedOffsetsAreConsidered.

@Test(timeout = 30000)
public void whenRebalanceThenPartitionsAreEquallyDistributedAndCommittedOffsetsAreConsidered() throws Exception {
    // write 5 events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah" + x, x -> String.valueOf(x % 8));
    // create a session
    final TestStreamingClient clientA = TestStreamingClient.create(URL, subscription.getId(), "max_uncommitted_events=100").start();
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(40)));
    // check that we received 5 events for each partitions
    range(0, 8).forEach(partition -> assertThat(clientA.getBatches().stream().filter(batch -> batch.getCursor().getPartition().equals(String.valueOf(partition))).count(), equalTo(5L)));
    // commit what we consumed
    final List<SubscriptionCursor> cursors = range(0, 8).boxed().map(partition -> new SubscriptionCursor(String.valueOf(partition), "4", eventType.getName(), "token")).collect(toList());
    commitCursors(subscription.getId(), cursors, clientA.getSessionId());
    // create second session for the same subscription
    final TestStreamingClient clientB = TestStreamingClient.create(URL, subscription.getId(), "stream_limit=20&max_uncommitted_events=100").start();
    // wait for rebalance process to finish
    waitFor(() -> assertThat(getNumberOfAssignedStreams(subscription.getId()), is(2)));
    // write 5 more events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah_" + x, x -> String.valueOf(x % 8));
    // wait till all events arrive
    waitFor(() -> assertThat(clientB.getBatches(), hasSize(20)));
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(60)));
    // check that only half of partitions were streamed to client A after rebalance
    final Set<String> clientAPartitionsAfterRebalance = getUniquePartitionsStreamedToClient(clientA, 40, 60);
    assertThat(clientAPartitionsAfterRebalance, hasSize(4));
    // check that only half of partitions were streamed to client B
    final Set<String> clientBPartitions = getUniquePartitionsStreamedToClient(clientB);
    assertThat(clientBPartitions, hasSize(4));
    // check that different partitions were streamed to different clients
    assertThat(intersection(clientAPartitionsAfterRebalance, clientBPartitions), hasSize(0));
    // commit what we consumed, as clientB has already consumed what was required by stream_limit - it should
    // be closed right after everything is committed
    final List<SubscriptionCursor> lastCursorsA = getLastCursorsForPartitions(clientA, clientAPartitionsAfterRebalance);
    final List<SubscriptionCursor> lastCursorsB = getLastCursorsForPartitions(clientB, clientBPartitions);
    commitCursors(subscription.getId(), lastCursorsA, clientA.getSessionId());
    commitCursors(subscription.getId(), lastCursorsB, clientB.getSessionId());
    waitFor(() -> assertThat(clientB.isRunning(), is(false)));
    // wait for rebalance process to finish
    waitFor(() -> assertThat(getNumberOfAssignedStreams(subscription.getId()), is(1)));
    // write 5 more events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah__" + x, x -> String.valueOf(x % 8));
    // check that after second rebalance all events were consumed by first client
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(100)));
}