Examples with PushOffsetVector io.confluent.ksql.util.PushOffsetVector used on opensource projects

Example 6 with PushOffsetVector

use of io.confluent.ksql.util.PushOffsetVector in project ksql by confluentinc.

the class ScalablePushConsumer method run.

public void run() {
    if (started) {
        LOG.error("Already ran consumer");
        throw new IllegalStateException("Already ran consumer");
    }
    started = true;
    try {
        initialize();
        subscribeOrAssign();
        while (!closed) {
            final ConsumerRecords<?, GenericRow> records = consumer.poll(POLL_TIMEOUT);
            // No assignment yet
            if (this.topicPartitions.get() == null) {
                continue;
            }
            if (newAssignment) {
                newAssignment = false;
                onNewAssignment();
            }
            final PushOffsetVector startOffsetVector = getOffsetVector(currentPositions.get(), topicName, partitions);
            if (records.isEmpty()) {
                updateCurrentPositions();
                computeProgressToken(Optional.of(startOffsetVector));
                onEmptyRecords();
                continue;
            }
            for (ConsumerRecord<?, GenericRow> rec : records) {
                handleRow(rec.key(), rec.value(), rec.timestamp());
            }
            updateCurrentPositions();
            computeProgressToken(Optional.of(startOffsetVector));
            try {
                consumer.commitSync();
            } catch (CommitFailedException e) {
                LOG.warn("Failed to commit, likely due to rebalance.  Will wait for new assignment", e);
            }
            afterBatchProcessed();
        }
    } catch (WakeupException e) {
    // This is expected when we get closed.
    }
}

Example 7 with PushOffsetVector

use of io.confluent.ksql.util.PushOffsetVector in project ksql by confluentinc.

the class CatchupConsumerTest method shouldRunConsumer_timeoutWaitingForAssignment.

@Test
public void shouldRunConsumer_timeoutWaitingForAssignment() {
    // Given:
    PushOffsetRange offsetRange = new PushOffsetRange(Optional.empty(), new PushOffsetVector(ImmutableList.of(1L, 2L)));
    when(latestConsumer.getAssignment()).thenReturn(null);
    when(clock.millis()).thenReturn(CURRENT_TIME_MS, CURRENT_TIME_MS + WAIT_FOR_ASSIGNMENT_MS - 1, CURRENT_TIME_MS + WAIT_FOR_ASSIGNMENT_MS + 1);
    try (CatchupConsumer consumer = new CatchupConsumer(TOPIC, false, SCHEMA, kafkaConsumer, () -> latestConsumer, catchupCoordinator, offsetRange, clock, sleepFn, waitFn, 0, pq -> caughtUp = true)) {
        // When:
        consumer.register(queue);
        final Exception e = assertThrows(KsqlException.class, consumer::run);
        // Then:
        assertThat(e.getMessage(), containsString("Timed out waiting for assignment from Latest"));
    }
}