Examples with RecordPublisher org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher used on opensource projects

Example 11 with RecordPublisher

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV1Records.

@Test
public void testToSdkV1Records() throws Exception {
    Date now = new Date();
    byte[] data = new byte[] { 0, 1, 2, 3 };
    Record record = Record.builder().approximateArrivalTimestamp(now.toInstant()).partitionKey("pk").sequenceNumber("sn").data(SdkBytes.fromByteArray(data)).build();
    KinesisProxyV2Interface kinesis = singletonShard(createSubscribeToShardEvent(record));
    RecordPublisher publisher = createRecordPublisher(kinesis, latest());
    TestConsumer consumer = new TestConsumer();
    publisher.run(consumer);
    UserRecord actual = consumer.getRecordBatches().get(0).getDeaggregatedRecords().get(0);
    assertFalse(actual.isAggregated());
    assertEquals(now, actual.getApproximateArrivalTimestamp());
    assertEquals("sn", actual.getSequenceNumber());
    assertEquals("pk", actual.getPartitionKey());
    assertThat(toByteArray(actual.getData()), Matchers.equalTo(data));
}

Example 12 with RecordPublisher

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionAtSequenceNumber.

@Test
public void testToSdkV2StartingPositionAtSequenceNumber() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.restartFromSequenceNumber(AGGREGATED_SEQUENCE_NUMBER));
    publisher.run(new TestConsumer());
    assertEquals(DUMMY_SEQUENCE, kinesis.getStartingPositionForSubscription(0).sequenceNumber());
    assertEquals(AT_SEQUENCE_NUMBER, kinesis.getStartingPositionForSubscription(0).type());
}

Example 13 with RecordPublisher

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher in project flink by apache.

the class FanOutRecordPublisherTest method testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription.

@Test
public void testShardConsumerRetriesIfLimitExceededExceptionThrownFromSubscription() throws Exception {
    LimitExceededException exception = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(exception);
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    TestConsumer consumer = new TestConsumer();
    RecordPublisherRunResult result = recordPublisher.run(consumer);
    // An exception is thrown after the 5th record in each subscription, therefore we expect to
    // receive 5 records
    assertEquals(5, consumer.getRecordBatches().size());
    assertEquals(1, kinesis.getNumberOfSubscribeToShardInvocations());
    // INCOMPLETE is returned to indicate the shard is not complete
    assertEquals(INCOMPLETE, result);
}

Example 14 with RecordPublisher

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionAfterSequenceNumber.

@Test
public void testToSdkV2StartingPositionAfterSequenceNumber() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.continueFromSequenceNumber(SEQUENCE_NUMBER));
    publisher.run(new TestConsumer());
    assertEquals(DUMMY_SEQUENCE, kinesis.getStartingPositionForSubscription(0).sequenceNumber());
    assertEquals(AFTER_SEQUENCE_NUMBER, kinesis.getStartingPositionForSubscription(0).type());
}

Example 15 with RecordPublisher

use of org.apache.flink.streaming.connectors.kinesis.internals.publisher.RecordPublisher in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV2StartingPositionAtTimeStamp.

@Test
public void testToSdkV2StartingPositionAtTimeStamp() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = emptyShard();
    Date now = new Date();
    RecordPublisher publisher = createRecordPublisher(kinesis, StartingPosition.fromTimestamp(now));
    publisher.run(new TestConsumer());
    assertEquals(now.toInstant(), kinesis.getStartingPositionForSubscription(0).timestamp());
    assertEquals(AT_TIMESTAMP, kinesis.getStartingPositionForSubscription(0).type());
}