Examples with StartingPosition org.apache.flink.streaming.connectors.kinesis.model.StartingPosition used on opensource projects

Example 1 with StartingPosition

use of org.apache.flink.streaming.connectors.kinesis.model.StartingPosition in project flink by apache.

the class AWSUtilTest method testGetStartingPositionForLatest.

@Test
public void testGetStartingPositionForLatest() {
    StartingPosition position = AWSUtil.getStartingPosition(SENTINEL_LATEST_SEQUENCE_NUM.get(), new Properties());
    assertEquals(AT_TIMESTAMP, position.getShardIteratorType());
    assertNotNull(position.getStartingMarker());
}

Example 2 with StartingPosition

use of org.apache.flink.streaming.connectors.kinesis.model.StartingPosition in project flink by apache.

the class ShardConsumerTestUtils method assertNumberOfMessagesReceivedFromKinesis.

public static ShardConsumerMetricsReporter assertNumberOfMessagesReceivedFromKinesis(final int expectedNumberOfMessages, final RecordPublisherFactory recordPublisherFactory, final SequenceNumber startingSequenceNumber, final Properties consumerProperties, final SequenceNumber expectedLastProcessedSequenceNum, final AbstractMetricGroup metricGroup) throws InterruptedException {
    ShardConsumerMetricsReporter shardMetricsReporter = new ShardConsumerMetricsReporter(metricGroup);
    StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0);
    LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
    subscribedShardsStateUnderTest.add(new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, startingSequenceNumber));
    TestSourceContext<String> sourceContext = new TestSourceContext<>();
    KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema());
    TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>(Collections.singletonList("fakeStream"), sourceContext, consumerProperties, deserializationSchema, 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class), Mockito.mock(KinesisProxyV2Interface.class));
    final StreamShardHandle shardHandle = subscribedShardsStateUnderTest.get(0).getStreamShardHandle();
    final SequenceNumber lastProcessedSequenceNum = subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum();
    final StartingPosition startingPosition = AWSUtil.getStartingPosition(lastProcessedSequenceNum, consumerProperties);
    final RecordPublisher recordPublisher = recordPublisherFactory.create(startingPosition, fetcher.getConsumerConfiguration(), metricGroup, shardHandle);
    int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0));
    new ShardConsumer<>(fetcher, recordPublisher, shardIndex, shardHandle, lastProcessedSequenceNum, shardMetricsReporter, deserializationSchema).run();
    assertEquals(expectedNumberOfMessages, sourceContext.getCollectedOutputs().size());
    assertEquals(expectedLastProcessedSequenceNum, subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum());
    return shardMetricsReporter;
}

Example 3 with StartingPosition

use of org.apache.flink.streaming.connectors.kinesis.model.StartingPosition in project flink by apache.

the class AWSUtilTest method testGetStartingPositionForTimestamp.

@Test
public void testGetStartingPositionForTimestamp() throws Exception {
    String timestamp = "2020-08-13T09:18:00.0+01:00";
    Date expectedTimestamp = new SimpleDateFormat(DEFAULT_STREAM_TIMESTAMP_DATE_FORMAT).parse(timestamp);
    Properties consumerProperties = new Properties();
    consumerProperties.setProperty(STREAM_INITIAL_TIMESTAMP, timestamp);
    StartingPosition position = AWSUtil.getStartingPosition(SENTINEL_AT_TIMESTAMP_SEQUENCE_NUM.get(), consumerProperties);
    assertEquals(AT_TIMESTAMP, position.getShardIteratorType());
    assertEquals(expectedTimestamp, position.getStartingMarker());
}