Examples with KinesisStreamShard org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard used on opensource projects

Example 1 with KinesisStreamShard

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard in project flink by apache.

the class FlinkKinesisConsumerTest method testLegacyKinesisStreamShardToStreamShardMetadataConversion.

@Test
public void testLegacyKinesisStreamShardToStreamShardMetadataConversion() {
    String streamName = "fakeStream1";
    String shardId = "shard-000001";
    String parentShardId = "shard-000002";
    String adjacentParentShardId = "shard-000003";
    String startingHashKey = "key-000001";
    String endingHashKey = "key-000010";
    String startingSequenceNumber = "seq-0000021";
    String endingSequenceNumber = "seq-00000031";
    StreamShardMetadata streamShardMetadata = new StreamShardMetadata();
    streamShardMetadata.setStreamName(streamName);
    streamShardMetadata.setShardId(shardId);
    streamShardMetadata.setParentShardId(parentShardId);
    streamShardMetadata.setAdjacentParentShardId(adjacentParentShardId);
    streamShardMetadata.setStartingHashKey(startingHashKey);
    streamShardMetadata.setEndingHashKey(endingHashKey);
    streamShardMetadata.setStartingSequenceNumber(startingSequenceNumber);
    streamShardMetadata.setEndingSequenceNumber(endingSequenceNumber);
    Shard shard = new Shard().withShardId(shardId).withParentShardId(parentShardId).withAdjacentParentShardId(adjacentParentShardId).withHashKeyRange(new HashKeyRange().withStartingHashKey(startingHashKey).withEndingHashKey(endingHashKey)).withSequenceNumberRange(new SequenceNumberRange().withStartingSequenceNumber(startingSequenceNumber).withEndingSequenceNumber(endingSequenceNumber));
    KinesisStreamShard kinesisStreamShard = new KinesisStreamShard(streamName, shard);
    assertEquals(streamShardMetadata, KinesisStreamShard.convertToStreamShardMetadata(kinesisStreamShard));
}

Example 2 with KinesisStreamShard

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShard in project flink by apache.

the class ShardConsumerTest method testCorrectNumOfCollectedRecordsAndUpdatedState.

@Test
public void testCorrectNumOfCollectedRecordsAndUpdatedState() {
    KinesisStreamShard fakeToBeConsumedShard = new KinesisStreamShard("fakeStream", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0)).withHashKeyRange(new HashKeyRange().withStartingHashKey("0").withEndingHashKey(new BigInteger(StringUtils.repeat("FF", 16), 16).toString())));
    LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>();
    subscribedShardsStateUnderTest.add(new KinesisStreamShardState(fakeToBeConsumedShard, new SequenceNumber("fakeStartingState")));
    TestableKinesisDataFetcher fetcher = new TestableKinesisDataFetcher(Collections.singletonList("fakeStream"), new Properties(), 10, 2, new AtomicReference<Throwable>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class));
    new ShardConsumer<>(fetcher, 0, subscribedShardsStateUnderTest.get(0).getKinesisStreamShard(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9)).run();
    assertTrue(fetcher.getNumOfElementsCollected() == 1000);
    assertTrue(subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get()));
}