Examples with GetShardListResult org.apache.flink.streaming.connectors.kinesis.proxy.GetShardListResult used on opensource projects

Example 1 with GetShardListResult

use of org.apache.flink.streaming.connectors.kinesis.proxy.GetShardListResult in project flink by apache.

the class KinesisDataFetcher method discoverNewShardsToSubscribe.

/**
	 * A utility function that does the following:
	 *
	 * 1. Find new shards for each stream that we haven't seen before
	 * 2. For each new shard, determine whether this consumer subtask should subscribe to them;
	 * 	  if yes, it is added to the returned list of shards
	 * 3. Update the subscribedStreamsToLastDiscoveredShardIds state so that we won't get shards
	 *    that we have already seen before the next time this function is called
	 */
private List<KinesisStreamShard> discoverNewShardsToSubscribe() throws InterruptedException {
    List<KinesisStreamShard> newShardsToSubscribe = new LinkedList<>();
    GetShardListResult shardListResult = kinesis.getShardList(subscribedStreamsToLastDiscoveredShardIds);
    if (shardListResult.hasRetrievedShards()) {
        Set<String> streamsWithNewShards = shardListResult.getStreamsWithRetrievedShards();
        for (String stream : streamsWithNewShards) {
            List<KinesisStreamShard> newShardsOfStream = shardListResult.getRetrievedShardListOfStream(stream);
            for (KinesisStreamShard newShard : newShardsOfStream) {
                if (isThisSubtaskShouldSubscribeTo(newShard, totalNumberOfConsumerSubtasks, indexOfThisConsumerSubtask)) {
                    newShardsToSubscribe.add(newShard);
                }
            }
            advanceLastDiscoveredShardOfStream(stream, shardListResult.getLastSeenShardOfStream(stream).getShard().getShardId());
        }
    }
    return newShardsToSubscribe;
}