Examples with KinesisStreamShardState org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState used on opensource projects

Example 6 with KinesisStreamShardState

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState in project flink by apache.

the class KinesisDataFetcher method runFetcher.

/**
	 * Starts the fetcher. After starting the fetcher, it can only
	 * be stopped by calling {@link KinesisDataFetcher#shutdownFetcher()}.
	 *
	 * @throws Exception the first error or exception thrown by the fetcher or any of the threads created by the fetcher.
	 */
public void runFetcher() throws Exception {
    // check that we are running before proceeding
    if (!running) {
        return;
    }
    this.mainThread = Thread.currentThread();
    //     and register them to the subscribedShardState list.
    if (LOG.isDebugEnabled()) {
        String logFormat = (!isRestoredFromFailure) ? "Subtask {} is trying to discover initial shards ..." : "Subtask {} is trying to discover any new shards that were created while the consumer wasn't " + "running due to failure ...";
        LOG.debug(logFormat, indexOfThisConsumerSubtask);
    }
    List<KinesisStreamShard> newShardsCreatedWhileNotRunning = discoverNewShardsToSubscribe();
    for (KinesisStreamShard shard : newShardsCreatedWhileNotRunning) {
        // the starting state for new shards created while the consumer wasn't running depends on whether or not
        // we are starting fresh (not restoring from a checkpoint); when we are starting fresh, this simply means
        // all existing shards of streams we are subscribing to are new shards; when we are restoring from checkpoint,
        // any new shards due to Kinesis resharding from the time of the checkpoint will be considered new shards.
        InitialPosition initialPosition = InitialPosition.valueOf(configProps.getProperty(ConsumerConfigConstants.STREAM_INITIAL_POSITION, ConsumerConfigConstants.DEFAULT_STREAM_INITIAL_POSITION));
        SentinelSequenceNumber startingStateForNewShard = (isRestoredFromFailure) ? SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM : initialPosition.toSentinelSequenceNumber();
        if (LOG.isInfoEnabled()) {
            String logFormat = (!isRestoredFromFailure) ? "Subtask {} will be seeded with initial shard {}, starting state set as sequence number {}" : "Subtask {} will be seeded with new shard {} that was created while the consumer wasn't " + "running due to failure, starting state set as sequence number {}";
            LOG.info(logFormat, indexOfThisConsumerSubtask, shard.toString(), startingStateForNewShard.get());
        }
        registerNewSubscribedShardState(new KinesisStreamShardState(shard, startingStateForNewShard.get()));
    }
    //  2. check that there is at least one shard in the subscribed streams to consume from (can be done by
    //     checking if at least one value in subscribedStreamsToLastDiscoveredShardIds is not null)
    boolean hasShards = false;
    StringBuilder streamsWithNoShardsFound = new StringBuilder();
    for (Map.Entry<String, String> streamToLastDiscoveredShardEntry : subscribedStreamsToLastDiscoveredShardIds.entrySet()) {
        if (streamToLastDiscoveredShardEntry.getValue() != null) {
            hasShards = true;
        } else {
            streamsWithNoShardsFound.append(streamToLastDiscoveredShardEntry.getKey()).append(", ");
        }
    }
    if (streamsWithNoShardsFound.length() != 0 && LOG.isWarnEnabled()) {
        LOG.warn("Subtask {} has failed to find any shards for the following subscribed streams: {}", indexOfThisConsumerSubtask, streamsWithNoShardsFound.toString());
    }
    if (!hasShards) {
        throw new RuntimeException("No shards can be found for all subscribed streams: " + streams);
    }
    //     consumer using a restored state checkpoint
    for (int seededStateIndex = 0; seededStateIndex < subscribedShardsState.size(); seededStateIndex++) {
        KinesisStreamShardState seededShardState = subscribedShardsState.get(seededStateIndex);
        // only start a consuming thread if the seeded subscribed shard has not been completely read already
        if (!seededShardState.getLastProcessedSequenceNum().equals(SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get())) {
            if (LOG.isInfoEnabled()) {
                LOG.info("Subtask {} will start consuming seeded shard {} from sequence number {} with ShardConsumer {}", indexOfThisConsumerSubtask, seededShardState.getKinesisStreamShard().toString(), seededShardState.getLastProcessedSequenceNum(), seededStateIndex);
            }
            shardConsumersExecutor.submit(new ShardConsumer<>(this, seededStateIndex, subscribedShardsState.get(seededStateIndex).getKinesisStreamShard(), subscribedShardsState.get(seededStateIndex).getLastProcessedSequenceNum()));
        }
    }
    // ------------------------------------------------------------------------
    // finally, start the infinite shard discovery and consumer launching loop;
    // we will escape from this loop only when shutdownFetcher() or stopWithError() is called
    final long discoveryIntervalMillis = Long.valueOf(configProps.getProperty(ConsumerConfigConstants.SHARD_DISCOVERY_INTERVAL_MILLIS, Long.toString(ConsumerConfigConstants.DEFAULT_SHARD_DISCOVERY_INTERVAL_MILLIS)));
    // FLINK-4341:
    // For downstream operators that work on time (ex. window operators), we are required to emit a max value watermark
    // for subtasks that won't continue to have shards to read from unless resharding happens in the future, otherwise
    // the downstream watermarks would not advance, leading to unbounded accumulating state.
    //
    // The side-effect of this limitation is that on resharding, we must fail hard if the newly discovered shard
    // is to be subscribed by a subtask that has previously emitted a max value watermark, otherwise the watermarks
    // will be messed up.
    //
    // There are 2 cases were we need to either emit a max value watermark, or deliberately fail hard:
    //  (a) if this subtask has no more shards to read from unless resharding happens in the future, we emit a max
    //      value watermark. This case is encountered when 1) all previously read shards by this subtask were closed
    //      due to resharding, 2) when this subtask was initially only subscribed to closed shards while the consumer
    //      was told to start from TRIM_HORIZON, or 3) there was initially no shards for this subtask to read on startup.
    //  (b) this subtask has discovered new shards to read from due to a reshard; if this subtask has already emitted
    //      a max value watermark, we must deliberately fail hard to avoid messing up the watermarks. The new shards
    //      will be subscribed by this subtask after restore as initial shards on startup.
    //
    // TODO: This is a temporary workaround until a min-watermark information service is available in the JobManager
    // Please see FLINK-4341 for more detail
    boolean emittedMaxValueWatermark = false;
    if (this.numberOfActiveShards.get() == 0) {
        // FLINK-4341 workaround case (a) - please see the above for details on this case
        LOG.info("Subtask {} has no initial shards to read on startup; emitting max value watermark ...", indexOfThisConsumerSubtask);
        sourceContext.emitWatermark(new Watermark(Long.MAX_VALUE));
        emittedMaxValueWatermark = true;
    }
    while (running) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Subtask {} is trying to discover new shards that were created due to resharding ...", indexOfThisConsumerSubtask);
        }
        List<KinesisStreamShard> newShardsDueToResharding = discoverNewShardsToSubscribe();
        if (newShardsDueToResharding.size() == 0 && this.numberOfActiveShards.get() == 0 && !emittedMaxValueWatermark) {
            // FLINK-4341 workaround case (a) - please see the above for details on this case
            LOG.info("Subtask {} has completed reading all shards; emitting max value watermark ...", indexOfThisConsumerSubtask);
            sourceContext.emitWatermark(new Watermark(Long.MAX_VALUE));
            emittedMaxValueWatermark = true;
        } else if (newShardsDueToResharding.size() > 0 && emittedMaxValueWatermark) {
            // FLINK-4341 workaround case (b) - please see the above for details on this case
            //
            // Note that in the case where on resharding this subtask ceased to read all of it's previous shards
            // but new shards is also to be subscribed by this subtask immediately after, emittedMaxValueWatermark
            // will be false; this allows the fetcher to continue reading the new shards without failing on such cases.
            // However, due to the race condition mentioned above, we might still fall into case (a) first, and
            // then (b) on the next discovery attempt. Although the failure is ideally unnecessary, max value
            // watermark emitting still remains to be correct.
            LOG.warn("Subtask {} has discovered {} new shards to subscribe, but is failing hard to avoid messing" + " up watermarks; the new shards will be subscribed by this subtask after restore ...", indexOfThisConsumerSubtask, newShardsDueToResharding.size());
            throw new RuntimeException("Deliberate failure to avoid messing up watermarks");
        }
        for (KinesisStreamShard shard : newShardsDueToResharding) {
            // since there may be delay in discovering a new shard, all new shards due to
            // resharding should be read starting from the earliest record possible
            KinesisStreamShardState newShardState = new KinesisStreamShardState(shard, SentinelSequenceNumber.SENTINEL_EARLIEST_SEQUENCE_NUM.get());
            int newStateIndex = registerNewSubscribedShardState(newShardState);
            if (LOG.isInfoEnabled()) {
                LOG.info("Subtask {} has discovered a new shard {} due to resharding, and will start consuming " + "the shard from sequence number {} with ShardConsumer {}", indexOfThisConsumerSubtask, newShardState.getKinesisStreamShard().toString(), newShardState.getLastProcessedSequenceNum(), newStateIndex);
            }
            shardConsumersExecutor.submit(new ShardConsumer<>(this, newStateIndex, newShardState.getKinesisStreamShard(), newShardState.getLastProcessedSequenceNum()));
        }
        // interval if the running flag was set to false during the middle of the while loop
        if (running && discoveryIntervalMillis != 0) {
            try {
                Thread.sleep(discoveryIntervalMillis);
            } catch (InterruptedException iex) {
            // the sleep may be interrupted by shutdownFetcher()
            }
        }
    }
    // make sure all resources have been terminated before leaving
    awaitTermination();
    // any error thrown in the shard consumer threads will be thrown to the main thread
    Throwable throwable = this.error.get();
    if (throwable != null) {
        if (throwable instanceof Exception) {
            throw (Exception) throwable;
        } else if (throwable instanceof Error) {
            throw (Error) throwable;
        } else {
            throw new Exception(throwable);
        }
    }
}

Example 7 with KinesisStreamShardState

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState in project flink by apache.

the class FlinkKinesisConsumerTest method testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint.

@Test
@SuppressWarnings("unchecked")
public void testFetcherShouldBeCorrectlySeededIfRestoringFromCheckpoint() throws Exception {
    KinesisDataFetcher mockedFetcher = Mockito.mock(KinesisDataFetcher.class);
    PowerMockito.whenNew(KinesisDataFetcher.class).withAnyArguments().thenReturn(mockedFetcher);
    // assume the given config is correct
    PowerMockito.mockStatic(KinesisConfigUtil.class);
    PowerMockito.doNothing().when(KinesisConfigUtil.class);
    HashMap<KinesisStreamShard, SequenceNumber> fakeRestoredState = new HashMap<>();
    fakeRestoredState.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), new SequenceNumber(UUID.randomUUID().toString()));
    fakeRestoredState.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), new SequenceNumber(UUID.randomUUID().toString()));
    fakeRestoredState.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))), new SequenceNumber(UUID.randomUUID().toString()));
    fakeRestoredState.put(new KinesisStreamShard("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), new SequenceNumber(UUID.randomUUID().toString()));
    fakeRestoredState.put(new KinesisStreamShard("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), new SequenceNumber(UUID.randomUUID().toString()));
    TestableFlinkKinesisConsumer consumer = new TestableFlinkKinesisConsumer("fakeStream", new Properties(), 10, 2);
    consumer.restoreState(fakeRestoredState);
    consumer.open(new Configuration());
    consumer.run(Mockito.mock(SourceFunction.SourceContext.class));
    Mockito.verify(mockedFetcher).setIsRestoringFromFailure(true);
    for (Map.Entry<KinesisStreamShard, SequenceNumber> restoredShard : fakeRestoredState.entrySet()) {
        Mockito.verify(mockedFetcher).advanceLastDiscoveredShardOfStream(restoredShard.getKey().getStreamName(), restoredShard.getKey().getShard().getShardId());
        Mockito.verify(mockedFetcher).registerNewSubscribedShardState(new KinesisStreamShardState(restoredShard.getKey(), restoredShard.getValue()));
    }
}

Example 8 with KinesisStreamShardState

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState in project flink by apache.

the class KinesisDataFetcherTest method testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceRestoredCheckpoint.

@Test
public void testStreamToLastSeenShardStateIsCorrectlySetWhenNewShardsFoundSinceRestoredCheckpoint() throws Exception {
    List<String> fakeStreams = new LinkedList<>();
    fakeStreams.add("fakeStream1");
    fakeStreams.add("fakeStream2");
    Map<KinesisStreamShard, String> restoredStateUnderTest = new HashMap<>();
    // fakeStream1 has 3 shards before restore
    restoredStateUnderTest.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new KinesisStreamShard("fakeStream1", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(2))), UUID.randomUUID().toString());
    // fakeStream2 has 2 shards before restore
    restoredStateUnderTest.put(new KinesisStreamShard("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(0))), UUID.randomUUID().toString());
    restoredStateUnderTest.put(new KinesisStreamShard("fakeStream2", new Shard().withShardId(KinesisShardIdGenerator.generateFromShardOrder(1))), UUID.randomUUID().toString());
    Map<String, Integer> streamToShardCount = new HashMap<>();
    // fakeStream1 had 3 shards before & 1 new shard after restore
    streamToShardCount.put("fakeStream1", 3 + 1);
    // fakeStream2 had 2 shards before & 3 new shard after restore
    streamToShardCount.put("fakeStream2", 2 + 3);
    HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest = KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
    // using a non-resharded streams kinesis behaviour to represent that Kinesis is not resharded AFTER the restore
    final TestableKinesisDataFetcher fetcher = new TestableKinesisDataFetcher(fakeStreams, new Properties(), 10, 2, new AtomicReference<Throwable>(), new LinkedList<KinesisStreamShardState>(), subscribedStreamsToLastSeenShardIdsUnderTest, FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
    for (Map.Entry<KinesisStreamShard, String> restoredState : restoredStateUnderTest.entrySet()) {
        fetcher.advanceLastDiscoveredShardOfStream(restoredState.getKey().getStreamName(), restoredState.getKey().getShard().getShardId());
        fetcher.registerNewSubscribedShardState(new KinesisStreamShardState(restoredState.getKey(), new SequenceNumber(restoredState.getValue())));
    }
    fetcher.setIsRestoringFromFailure(true);
    PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
    Thread runFetcherThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                fetcher.runFetcher();
            } catch (Exception e) {
            //
            }
        }
    });
    runFetcherThread.start();
    // sleep a while before closing
    Thread.sleep(1000);
    fetcher.shutdownFetcher();
    // assert that the streams tracked in the state are identical to the subscribed streams
    Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
    assertTrue(streamsInState.size() == fakeStreams.size());
    assertTrue(streamsInState.containsAll(fakeStreams));
    // assert that the last seen shards in state is correctly set
    for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
        assertTrue(streamToLastSeenShard.getValue().equals(KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
    }
}

Example 9 with KinesisStreamShardState

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState in project flink by apache.

the class KinesisDataFetcherTest method testStreamToLastSeenShardStateIsCorrectlySetWhenNotRestoringFromFailure.

@Test
public void testStreamToLastSeenShardStateIsCorrectlySetWhenNotRestoringFromFailure() throws Exception {
    List<String> fakeStreams = new LinkedList<>();
    fakeStreams.add("fakeStream1");
    fakeStreams.add("fakeStream2");
    fakeStreams.add("fakeStream3");
    fakeStreams.add("fakeStream4");
    HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest = KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
    Map<String, Integer> streamToShardCount = new HashMap<>();
    Random rand = new Random();
    for (String fakeStream : fakeStreams) {
        streamToShardCount.put(fakeStream, rand.nextInt(5) + 1);
    }
    final TestableKinesisDataFetcher fetcher = new TestableKinesisDataFetcher(fakeStreams, new Properties(), 10, 2, new AtomicReference<Throwable>(), new LinkedList<KinesisStreamShardState>(), subscribedStreamsToLastSeenShardIdsUnderTest, FakeKinesisBehavioursFactory.nonReshardedStreamsBehaviour(streamToShardCount));
    fetcher.setIsRestoringFromFailure(false);
    PowerMockito.whenNew(ShardConsumer.class).withAnyArguments().thenReturn(Mockito.mock(ShardConsumer.class));
    Thread runFetcherThread = new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                fetcher.runFetcher();
            } catch (Exception e) {
            //
            }
        }
    });
    runFetcherThread.start();
    // sleep a while before closing
    Thread.sleep(1000);
    fetcher.shutdownFetcher();
    // assert that the streams tracked in the state are identical to the subscribed streams
    Set<String> streamsInState = subscribedStreamsToLastSeenShardIdsUnderTest.keySet();
    assertTrue(streamsInState.size() == fakeStreams.size());
    assertTrue(streamsInState.containsAll(fakeStreams));
    // assert that the last seen shards in state is correctly set
    for (Map.Entry<String, String> streamToLastSeenShard : subscribedStreamsToLastSeenShardIdsUnderTest.entrySet()) {
        assertTrue(streamToLastSeenShard.getValue().equals(KinesisShardIdGenerator.generateFromShardOrder(streamToShardCount.get(streamToLastSeenShard.getKey()) - 1)));
    }
}

Example 10 with KinesisStreamShardState

use of org.apache.flink.streaming.connectors.kinesis.model.KinesisStreamShardState in project flink by apache.

the class KinesisDataFetcherTest method testIfNoShardsAreFoundShouldThrowException.

@Test(expected = RuntimeException.class)
public void testIfNoShardsAreFoundShouldThrowException() throws Exception {
    List<String> fakeStreams = new LinkedList<>();
    fakeStreams.add("fakeStream1");
    fakeStreams.add("fakeStream2");
    HashMap<String, String> subscribedStreamsToLastSeenShardIdsUnderTest = KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(fakeStreams);
    TestableKinesisDataFetcher fetcher = new TestableKinesisDataFetcher(fakeStreams, new Properties(), 10, 2, new AtomicReference<Throwable>(), new LinkedList<KinesisStreamShardState>(), subscribedStreamsToLastSeenShardIdsUnderTest, FakeKinesisBehavioursFactory.noShardsFoundForRequestedStreamsBehaviour());
    // not restoring
    fetcher.setIsRestoringFromFailure(false);
    // this should throw RuntimeException
    fetcher.runFetcher();
}