Examples with TestConsumer org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer used on opensource projects

Example 11 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class PollingRecordPublisherTest method testRunEmitsRunLoopTimeNanos.

@Test
public void testRunEmitsRunLoopTimeNanos() throws Exception {
    PollingRecordPublisherMetricsReporter metricsReporter = spy(new PollingRecordPublisherMetricsReporter(createFakeShardConsumerMetricGroup()));
    KinesisProxyInterface fakeKinesis = totalNumOfRecordsAfterNumOfGetRecordsCalls(5, 5, 100);
    PollingRecordPublisher recordPublisher = createPollingRecordPublisher(fakeKinesis, metricsReporter);
    recordPublisher.run(new TestConsumer());
    // Expect that the run loop took at least FETCH_INTERVAL_MILLIS in nanos
    verify(metricsReporter).setRunLoopTimeNanos(geq(FETCH_INTERVAL_MILLIS * 1_000_000));
}

Example 12 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testSubscribeToShardFailsWhenMaxRetriesExceeded.

@Test
public void testSubscribeToShardFailsWhenMaxRetriesExceeded() throws Exception {
    thrown.expect(RuntimeException.class);
    thrown.expectMessage("Maximum retries exceeded for SubscribeToShard. Failed 3 times.");
    Properties efoProperties = createEfoProperties();
    efoProperties.setProperty(SUBSCRIBE_TO_SHARD_RETRIES, String.valueOf(EXPECTED_SUBSCRIBE_TO_SHARD_RETRIES));
    FanOutRecordPublisherConfiguration configuration = new FanOutRecordPublisherConfiguration(efoProperties, emptyList());
    LimitExceededException retryableError = LimitExceededException.builder().build();
    SubscriptionErrorKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.errorDuringSubscription(retryableError);
    FullJitterBackoff backoff = mock(FullJitterBackoff.class);
    FanOutRecordPublisher recordPublisher = new FanOutRecordPublisher(latest(), "arn", createDummyStreamShardHandle(), kinesis, configuration, backoff);
    int count = 0;
    while (recordPublisher.run(new TestConsumer()) == INCOMPLETE) {
        if (++count > EXPECTED_SUBSCRIBE_TO_SHARD_RETRIES) {
            break;
        }
    }
}

Example 13 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testAggregatedRecordDurability.

@Test
public void testAggregatedRecordDurability() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().withBatchCount(10).withAggregationFactor(5).withRecordsPerBatch(12).build();
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    TestConsumer consumer = new TestConsumer();
    int count = 0;
    while (recordPublisher.run(consumer) == INCOMPLETE) {
        if (++count > 5) {
            break;
        }
    }
    List<UserRecord> userRecords = flattenToUserRecords(consumer.getRecordBatches());
    // Should have received 10 * 12 * 5 = 600 records
    assertEquals(600, userRecords.size());
    int sequence = 1;
    long subsequence = 0;
    for (UserRecord userRecord : userRecords) {
        assertEquals(String.valueOf(sequence), userRecord.getSequenceNumber());
        assertEquals(subsequence++, userRecord.getSubSequenceNumber());
        if (subsequence == 5) {
            sequence++;
            subsequence = 0;
        }
    }
}

Example 14 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testRecordDurability.

@Test
public void testRecordDurability() throws Exception {
    SingleShardFanOutKinesisV2 kinesis = FakeKinesisFanOutBehavioursFactory.boundedShard().withBatchCount(10).withBatchesPerSubscription(3).withRecordsPerBatch(12).build();
    RecordPublisher recordPublisher = createRecordPublisher(kinesis);
    TestConsumer consumer = new TestConsumer();
    int count = 0;
    while (recordPublisher.run(consumer) == INCOMPLETE) {
        if (++count > 4) {
            break;
        }
    }
    List<UserRecord> userRecords = flattenToUserRecords(consumer.getRecordBatches());
    // Should have received 10 * 12 = 120 records
    assertEquals(120, userRecords.size());
    int expectedSequenceNumber = 1;
    for (UserRecord record : userRecords) {
        assertEquals(String.valueOf(expectedSequenceNumber++), record.getSequenceNumber());
    }
}

Example 15 with TestConsumer

use of org.apache.flink.streaming.connectors.kinesis.testutils.TestUtils.TestConsumer in project flink by apache.

the class FanOutRecordPublisherTest method testToSdkV1Records.

@Test
public void testToSdkV1Records() throws Exception {
    Date now = new Date();
    byte[] data = new byte[] { 0, 1, 2, 3 };
    Record record = Record.builder().approximateArrivalTimestamp(now.toInstant()).partitionKey("pk").sequenceNumber("sn").data(SdkBytes.fromByteArray(data)).build();
    KinesisProxyV2Interface kinesis = singletonShard(createSubscribeToShardEvent(record));
    RecordPublisher publisher = createRecordPublisher(kinesis, latest());
    TestConsumer consumer = new TestConsumer();
    publisher.run(consumer);
    UserRecord actual = consumer.getRecordBatches().get(0).getDeaggregatedRecords().get(0);
    assertFalse(actual.isAggregated());
    assertEquals(now, actual.getApproximateArrivalTimestamp());
    assertEquals("sn", actual.getSequenceNumber());
    assertEquals("pk", actual.getPartitionKey());
    assertThat(toByteArray(actual.getData()), Matchers.equalTo(data));
}