Examples with TestStreamingClient org.zalando.nakadi.webservice.utils.TestStreamingClient used on opensource projects

Example 21 with TestStreamingClient

use of org.zalando.nakadi.webservice.utils.TestStreamingClient in project nakadi by zalando.

the class HilaAT method whenStreamTimeoutReachedPossibleToCommit.

@Test(timeout = 10000)
public void whenStreamTimeoutReachedPossibleToCommit() throws Exception {
    final TestStreamingClient client = TestStreamingClient.create(URL, subscription.getId(), "batch_limit=1&stream_limit=2&stream_timeout=1").start();
    waitFor(() -> assertThat(client.getSessionId(), not(equalTo(SESSION_ID_UNKNOWN))));
    publishEvent(eventType.getName(), "{\"foo\":\"bar\"}");
    waitFor(() -> Assert.assertFalse(client.getBatches().isEmpty()), TimeUnit.SECONDS.toMillis(2), 100);
    final SubscriptionCursor toCommit = client.getBatches().get(0).getCursor();
    // connection is closed, and stream as well
    client.close();
    Thread.sleep(TimeUnit.SECONDS.toMillis(1));
    final int statusCode = commitCursors(subscription.getId(), Collections.singletonList(toCommit), client.getSessionId());
    Assert.assertEquals(SC_NO_CONTENT, statusCode);
}

Example 22 with TestStreamingClient

use of org.zalando.nakadi.webservice.utils.TestStreamingClient in project nakadi by zalando.

the class HilaAT method whenNoEventsThenBeginOffsetIsUsed.

@Test(timeout = 5000)
public void whenNoEventsThenBeginOffsetIsUsed() throws Exception {
    final TestStreamingClient client = TestStreamingClient.create(subscription.getId()).start();
    waitFor(() -> assertThat(client.getSessionId(), not(equalTo(SESSION_ID_UNKNOWN))));
    when().get("/subscriptions/{sid}/cursors", subscription.getId()).then().body("items[0].offset", equalTo("001-0001--1"));
    final int commitResult = commitCursors(subscription.getId(), ImmutableList.of(new SubscriptionCursor("0", Cursor.BEFORE_OLDEST_OFFSET, eventType.getName(), "abc")), client.getSessionId());
    assertThat(commitResult, equalTo(SC_OK));
}

Example 23 with TestStreamingClient

use of org.zalando.nakadi.webservice.utils.TestStreamingClient in project nakadi by zalando.

the class HilaAT method whenConnectionIsClosedByClientNakadiRecognizesIt.

@Test(timeout = 10000)
public void whenConnectionIsClosedByClientNakadiRecognizesIt() throws Exception {
    final TestStreamingClient client = TestStreamingClient.create(URL, subscription.getId(), "batch_flush_timeout=1");
    client.start();
    waitFor(() -> assertThat(client.getBatches(), hasSize(1)));
    client.close();
    Thread.sleep(2500);
    final TestStreamingClient anotherClient = TestStreamingClient.create(URL, subscription.getId(), "batch_flush_timeout=1");
    anotherClient.start();
    // if we start to get data for another client it means that Nakadi recognized that first client closed
    // connection (in other case it would not allow second client to connect because of lack of slots)
    waitFor(() -> assertThat(anotherClient.getBatches(), hasSize(1)));
}

Example 24 with TestStreamingClient

use of org.zalando.nakadi.webservice.utils.TestStreamingClient in project nakadi by zalando.

the class HilaRebalanceAT method checkDirectAssignmentCorrectlyCapturesAndReleasesPartition.

@Test(timeout = 15000)
public void checkDirectAssignmentCorrectlyCapturesAndReleasesPartition() throws IOException, InterruptedException {
    // launch two clients: one using auto-rebalance, second one directly reading from partition 6
    final TestStreamingClient autoClient = new TestStreamingClient(URL, subscription.getId(), "max_uncommitted_events=100");
    autoClient.start();
    final TestStreamingClient directClient = new TestStreamingClient(URL, subscription.getId(), "", Optional.empty(), Optional.of("{\"stream_limit\":1,\"partitions\":[" + "{\"event_type\":\"" + eventType.getName() + "\",\"partition\":\"6\"}]}"));
    directClient.start();
    // wait for rebalance to finish and send 1 event to each partition
    waitFor(() -> assertThat(getNumberOfAssignedStreams(subscription.getId()), is(2)));
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 8, x -> "blah" + x, x -> String.valueOf(x % 8));
    waitFor(() -> assertThat(autoClient.getBatches(), hasSize(7)));
    checkAllEventsAreFromPartitions(autoClient, ImmutableSet.of("0", "1", "2", "3", "4", "5", "7"));
    waitFor(() -> assertThat(directClient.getBatches(), hasSize(1)));
    checkAllEventsAreFromPartitions(directClient, ImmutableSet.of("6"));
    // commit cursors and wait for stream to be closed (because of reaching stream_limit)
    commitCursors(subscription.getId(), directClient.getBatches().stream().map(StreamBatch::getCursor).collect(Collectors.toList()), directClient.getSessionId());
    waitFor(() -> assertThat(directClient.isRunning(), is(false)));
    // send 1 event to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 8, x -> "blah" + x, x -> String.valueOf(x % 8));
    // the client with auto-balancing should now get all 8 new events
    waitFor(() -> assertThat(autoClient.getBatches(), hasSize(7 + 8)));
    checkAllEventsAreFromPartitions(autoClient, ImmutableSet.of("0", "1", "2", "3", "4", "5", "6", "7"));
}

Example 25 with TestStreamingClient

use of org.zalando.nakadi.webservice.utils.TestStreamingClient in project nakadi by zalando.

the class HilaRebalanceAT method whenRebalanceThenPartitionsAreEquallyDistributedAndCommittedOffsetsAreConsidered.

@Test(timeout = 30000)
public void whenRebalanceThenPartitionsAreEquallyDistributedAndCommittedOffsetsAreConsidered() throws Exception {
    // write 5 events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah" + x, x -> String.valueOf(x % 8));
    // create a session
    final TestStreamingClient clientA = TestStreamingClient.create(URL, subscription.getId(), "max_uncommitted_events=100").start();
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(40)));
    // check that we received 5 events for each partitions
    range(0, 8).forEach(partition -> assertThat(clientA.getBatches().stream().filter(batch -> batch.getCursor().getPartition().equals(String.valueOf(partition))).count(), equalTo(5L)));
    // commit what we consumed
    final List<SubscriptionCursor> cursors = range(0, 8).boxed().map(partition -> new SubscriptionCursor(String.valueOf(partition), "4", eventType.getName(), "token")).collect(toList());
    commitCursors(subscription.getId(), cursors, clientA.getSessionId());
    // create second session for the same subscription
    final TestStreamingClient clientB = TestStreamingClient.create(URL, subscription.getId(), "stream_limit=20&max_uncommitted_events=100").start();
    // wait for rebalance process to finish
    waitFor(() -> assertThat(getNumberOfAssignedStreams(subscription.getId()), is(2)));
    // write 5 more events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah_" + x, x -> String.valueOf(x % 8));
    // wait till all events arrive
    waitFor(() -> assertThat(clientB.getBatches(), hasSize(20)));
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(60)));
    // check that only half of partitions were streamed to client A after rebalance
    final Set<String> clientAPartitionsAfterRebalance = getUniquePartitionsStreamedToClient(clientA, 40, 60);
    assertThat(clientAPartitionsAfterRebalance, hasSize(4));
    // check that only half of partitions were streamed to client B
    final Set<String> clientBPartitions = getUniquePartitionsStreamedToClient(clientB);
    assertThat(clientBPartitions, hasSize(4));
    // check that different partitions were streamed to different clients
    assertThat(intersection(clientAPartitionsAfterRebalance, clientBPartitions), hasSize(0));
    // commit what we consumed, as clientB has already consumed what was required by stream_limit - it should
    // be closed right after everything is committed
    final List<SubscriptionCursor> lastCursorsA = getLastCursorsForPartitions(clientA, clientAPartitionsAfterRebalance);
    final List<SubscriptionCursor> lastCursorsB = getLastCursorsForPartitions(clientB, clientBPartitions);
    commitCursors(subscription.getId(), lastCursorsA, clientA.getSessionId());
    commitCursors(subscription.getId(), lastCursorsB, clientB.getSessionId());
    waitFor(() -> assertThat(clientB.isRunning(), is(false)));
    // wait for rebalance process to finish
    waitFor(() -> assertThat(getNumberOfAssignedStreams(subscription.getId()), is(1)));
    // write 5 more events to each partition
    publishBusinessEventWithUserDefinedPartition(eventType.getName(), 40, x -> "blah__" + x, x -> String.valueOf(x % 8));
    // check that after second rebalance all events were consumed by first client
    waitFor(() -> assertThat(clientA.getBatches(), hasSize(100)));
}