Examples with SingleShardFanOutKinesisV2 org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 used on opensource projects

Example 6 with SingleShardFanOutKinesisV2

use of org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionTrimHorizon.

@Test
public void testToSdkV2StartingPositionTrimHorizon() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.continueFromSequenceNumber(SENTINEL_EARLIEST_SEQUENCE_NUM.get()));
    publisher.run(new TestConsumer());
    assertNull(kinesis.getStartingPositionForSubscription(0).sequenceNumber());
    assertEquals(TRIM_HORIZON, kinesis.getStartingPositionForSubscription(0).type());
}

Example 7 with SingleShardFanOutKinesisV2

use of org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 in project flink by apache.

the class FanOutRecordPublisherTest method testExceptionThrownInConsumerPropagatesToRecordPublisher.

@Test
public void testExceptionThrownInConsumerPropagatesToRecordPublisher() throws Exception {
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("An error thrown from the consumer");
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().build();
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    recordPublisher.run(batch -> {
        throw new RuntimeException("An error thrown from the consumer");
    });
}

Example 8 with SingleShardFanOutKinesisV2

use of org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 in project flink by apache.

the class ShardConsumerFanOutTest method testSubscribeToShardUsesCorrectStartingSequenceNumbers.

@Test
public void testSubscribeToShardUsesCorrectStartingSequenceNumbers() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().withBatchCount(10).withRecordsPerBatch(1).withBatchesPerSubscription(2).build();
    // 10 batches of 1 records = 10
    assertNumberOfMessagesReceivedFromKinesis(10, kinesis, new SequenceNumber("0"));
    // 10 batches / 2 batches per subscription = 5 subscriptions
    assertEquals(5, kinesis.getNumberOfSubscribeToShardInvocations());
    // Starting positions should correlate to the last consumed sequence number
    assertStartingPositionAfterSequenceNumber(kinesis.getStartingPositionForSubscription(0), "0");
    assertStartingPositionAfterSequenceNumber(kinesis.getStartingPositionForSubscription(1), "2");
    assertStartingPositionAfterSequenceNumber(kinesis.getStartingPositionForSubscription(2), "4");
    assertStartingPositionAfterSequenceNumber(kinesis.getStartingPositionForSubscription(3), "6");
    assertStartingPositionAfterSequenceNumber(kinesis.getStartingPositionForSubscription(4), "8");
}

Example 9 with SingleShardFanOutKinesisV2

use of org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 in project flink by apache.

the class ShardConsumerFanOutTest method testMillisBehindReported.

@Test
public void testMillisBehindReported() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().withMillisBehindLatest(123L).build();
    // Fake behaviour defaults to 10 messages
    ShardConsumerMetricsReporter metrics = assertNumberOfMessagesReceivedFromKinesis(10, kinesis, fakeSequenceNumber());
    assertEquals(123L, metrics.getMillisBehindLatest());
}

Example 10 with SingleShardFanOutKinesisV2

use of org.apache.flink.streaming.connectors.kinesis.testutils.FakeKinesisFanOutBehavioursFactory.SingleShardFanOutKinesisV2 in project flink by apache.

the class ShardConsumerFanOutTest method testBoundedShardResubscribesToShard.

@Test
public void testBoundedShardResubscribesToShard() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().withBatchCount(100).withRecordsPerBatch(10).withBatchesPerSubscription(5).build();
    // 100 batches of 10 records = 1000
    assertNumberOfMessagesReceivedFromKinesis(1000, kinesis, fakeSequenceNumber());
    // 100 batches / 5 batches per subscription = 20 subscriptions
    assertEquals(20, kinesis.getNumberOfSubscribeToShardInvocations());
    // Starting from non-aggregated sequence number means we should start AFTER the sequence
    // number
    assertEquals(AFTER_SEQUENCE_NUMBER, kinesis.getStartingPositionForSubscription(0).type());
}