Examples with RecordBatch org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordBatch used on opensource projects

Example 1 with RecordBatch

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordBatch in project flink by apache.

the class PollingRecordPublisher method run.

public RecordPublisherRunResult run(final RecordBatchConsumer consumer, int maxNumberOfRecords) throws InterruptedException {
    if (nextShardItr == null) {
        return COMPLETE;
    }
    metricsReporter.setMaxNumberOfRecordsPerFetch(maxNumberOfRecords);
    GetRecordsResult result = getRecords(nextShardItr, maxNumberOfRecords);
    RecordBatch recordBatch = new RecordBatch(result.getRecords(), subscribedShard, result.getMillisBehindLatest());
    SequenceNumber latestSequenceNumber = consumer.accept(recordBatch);
    nextStartingPosition = getNextStartingPosition(latestSequenceNumber);
    nextShardItr = result.getNextShardIterator();
    long adjustmentEndTimeNanos = adjustRunLoopFrequency(processingStartTimeNanos, System.nanoTime());
    long runLoopTimeNanos = adjustmentEndTimeNanos - processingStartTimeNanos;
    processingStartTimeNanos = adjustmentEndTimeNanos;
    metricsReporter.setRunLoopTimeNanos(runLoopTimeNanos);
    return nextShardItr == null ? COMPLETE : INCOMPLETE;
}

Example 2 with RecordBatch

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordBatch in project flink by apache.

the class FanOutRecordPublisher method run.

@Override
public RecordPublisherRunResult run(final RecordBatchConsumer recordConsumer) throws InterruptedException {
    LOG.info("Running fan out record publisher on {}::{} from {} - {}", subscribedShard.getStreamName(), subscribedShard.getShard().getShardId(), nextStartingPosition.getShardIteratorType(), nextStartingPosition.getStartingMarker());
    Consumer<SubscribeToShardEvent> eventConsumer = event -> {
        RecordBatch recordBatch = new RecordBatch(toSdkV1Records(event.records()), subscribedShard, event.millisBehindLatest());
        SequenceNumber sequenceNumber = recordConsumer.accept(recordBatch);
        nextStartingPosition = StartingPosition.continueFromSequenceNumber(sequenceNumber);
    };
    RecordPublisherRunResult result = runWithBackoff(eventConsumer);
    LOG.info("Subscription expired {}::{}, with status {}", subscribedShard.getStreamName(), subscribedShard.getShard().getShardId(), result);
    return result;
}