Examples with StreamBatch org.zalando.nakadi.webservice.hila.StreamBatch used on opensource projects

Example 1 with StreamBatch

use of org.zalando.nakadi.webservice.hila.StreamBatch in project nakadi by zalando.

the class UserJourneyAT method userJourneyHila.

@Test(timeout = 15000)
public void userJourneyHila() throws InterruptedException, IOException {
    postEvents(rangeClosed(0, 3).mapToObj(x -> "{\"foo\":\"bar" + x + "\"}").collect(Collectors.toList()).toArray(new String[4]));
    // create subscription
    final SubscriptionBase subscriptionToCreate = RandomSubscriptionBuilder.builder().withOwningApplication("stups_aruha-test-end2end-nakadi").withEventType(eventTypeName).withStartFrom(BEGIN).buildSubscriptionBase();
    final Subscription subscription = createSubscription(jsonRequestSpec(), subscriptionToCreate);
    // list subscriptions
    jsonRequestSpec().param("event_type", eventTypeName).get("/subscriptions").then().statusCode(OK.value()).body("items.size()", equalTo(1)).body("items[0].id", equalTo(subscription.getId()));
    // create client and wait till we receive all events
    final TestStreamingClient client = new TestStreamingClient(RestAssured.baseURI + ":" + RestAssured.port, subscription.getId(), "", oauthToken).start();
    waitFor(() -> assertThat(client.getBatches(), hasSize(4)));
    final List<StreamBatch> batches = client.getBatches();
    // validate the content of events
    for (int i = 0; i < batches.size(); i++) {
        final SubscriptionCursor cursor = new SubscriptionCursor("0", TestUtils.toTimelineOffset(i), eventTypeName, "");
        final StreamBatch expectedBatch = new StreamBatch(cursor, ImmutableList.of(ImmutableMap.of("foo", "bar" + i)), i == 0 ? new StreamMetadata("Stream started") : null);
        final StreamBatch batch = batches.get(i);
        assertThat(batch, equalToBatchIgnoringToken(expectedBatch));
    }
    // as we didn't commit, there should be still 4 unconsumed events
    jsonRequestSpec().get("/subscriptions/{sid}/stats", subscription.getId()).then().statusCode(OK.value()).body("items[0].partitions[0].unconsumed_events", equalTo(4));
    // commit cursor of latest event
    final StreamBatch lastBatch = batches.get(batches.size() - 1);
    final int commitCode = commitCursors(jsonRequestSpec(), subscription.getId(), ImmutableList.of(lastBatch.getCursor()), client.getSessionId());
    assertThat(commitCode, equalTo(NO_CONTENT.value()));
    // now there should be 0 unconsumed events
    jsonRequestSpec().get("/subscriptions/{sid}/stats", subscription.getId()).then().statusCode(OK.value()).body("items[0].partitions[0].unconsumed_events", equalTo(0));
    // get cursors
    jsonRequestSpec().get("/subscriptions/{sid}/cursors", subscription.getId()).then().statusCode(OK.value()).body("items[0].partition", equalTo("0")).body("items[0].offset", equalTo("001-0001-000000000000000003"));
    // delete subscription
    jsonRequestSpec().delete("/subscriptions/{sid}", subscription.getId()).then().statusCode(NO_CONTENT.value());
}

Example 2 with StreamBatch

use of org.zalando.nakadi.webservice.hila.StreamBatch in project nakadi by zalando.

the class TestStreamingClient method run.

@Override
public void run() {
    LOG.info("Started streaming client thread");
    try {
        final String url = format("{0}/subscriptions/{1}/events?{2}", baseUrl, subscriptionId, params);
        connection = (HttpURLConnection) new URL(url).openConnection();
        token.ifPresent(token -> connection.setRequestProperty("Authorization", "Bearer " + token));
        if (bodyParams.isPresent()) {
            connection.setRequestMethod("POST");
            connection.setDoOutput(true);
            connection.setRequestProperty("Content-Type", "application/json");
            try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) {
                wr.write(bodyParams.get().getBytes(Charsets.UTF_8));
                wr.flush();
            }
        }
        responseCode = connection.getResponseCode();
        connection.getHeaderFields().entrySet().stream().filter(entry -> entry.getKey() != null).forEach(entry -> headers.put(entry.getKey(), entry.getValue()));
        connection.setReadTimeout(10);
        if (responseCode != HttpURLConnection.HTTP_OK) {
            throw new IOException("Response code is " + responseCode);
        }
        started.countDown();
        sessionId = connection.getHeaderField("X-Nakadi-StreamId");
        final InputStream inputStream = connection.getInputStream();
        final BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
        while (running) {
            try {
                final String line = reader.readLine();
                if (line == null) {
                    return;
                }
                final StreamBatch streamBatch = MAPPER.readValue(line, StreamBatch.class);
                synchronized (batches) {
                    batches.add(streamBatch);
                }
            } catch (final SocketTimeoutException ste) {
                LOG.info("No data in 10 ms, retrying read data");
            }
        }
    } catch (IOException e) {
        LOG.error(e.getMessage(), e);
    } finally {
        if (null != batchesListener) {
            batchesListener.accept(batches);
        }
        close();
    }
}

Example 3 with StreamBatch

use of org.zalando.nakadi.webservice.hila.StreamBatch in project nakadi by zalando.

the class TestStreamingClient method startWithAutocommit.

public TestStreamingClient startWithAutocommit(final Consumer<List<StreamBatch>> batchesListener) throws InterruptedException {
    this.batchesListener = batchesListener;
    final TestStreamingClient client = startInternal(true);
    final Thread autocommitThread = new Thread(() -> {
        int oldIdx = 0;
        while (client.isRunning()) {
            while (oldIdx < client.getBatches().size()) {
                final StreamBatch batch = client.getBatches().get(oldIdx);
                if (batch.getEvents() != null && !batch.getEvents().isEmpty()) {
                    try {
                        final SubscriptionCursor cursor = batch.getCursor();
                        final int responseCode = NakadiTestUtils.commitCursors(client.subscriptionId, Collections.singletonList(batch.getCursor()), client.getSessionId());
                        LOG.info("Committing " + responseCode + ": " + cursor);
                    } catch (JsonProcessingException e) {
                        throw new RuntimeException(e);
                    }
                }
                oldIdx += 1;
            }
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
    });
    autocommitThread.setDaemon(true);
    autocommitThread.start();
    return client;
}