Examples with StreamSource org.apache.flink.streaming.api.operators.StreamSource used on opensource projects

Example 11 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class SourceStreamTaskTest method testCheckpointing.

/**
	 * This test ensures that the SourceStreamTask properly serializes checkpointing
	 * and element emission. This also verifies that there are no concurrent invocations
	 * of the checkpoint method on the source operator.
	 *
	 * The source emits elements and performs checkpoints. We have several checkpointer threads
	 * that fire checkpoint requests at the source task.
	 *
	 * If element emission and checkpointing are not in series the count of elements at the
	 * beginning of a checkpoint and at the end of a checkpoint are not the same because the
	 * source kept emitting elements while the checkpoint was ongoing.
	 */
@Test
@SuppressWarnings("unchecked")
public void testCheckpointing() throws Exception {
    final int NUM_ELEMENTS = 100;
    final int NUM_CHECKPOINTS = 100;
    final int NUM_CHECKPOINTERS = 1;
    // in ms
    final int CHECKPOINT_INTERVAL = 5;
    // how many random values we sum up in storeCheckpoint
    final int SOURCE_CHECKPOINT_DELAY = 1000;
    // in ms
    final int SOURCE_READ_DELAY = 1;
    ExecutorService executor = Executors.newFixedThreadPool(10);
    try {
        final TupleTypeInfo<Tuple2<Long, Integer>> typeInfo = new TupleTypeInfo<Tuple2<Long, Integer>>(BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO);
        final SourceStreamTask<Tuple2<Long, Integer>, SourceFunction<Tuple2<Long, Integer>>, StreamSource<Tuple2<Long, Integer>, SourceFunction<Tuple2<Long, Integer>>>> sourceTask = new SourceStreamTask<>();
        final StreamTaskTestHarness<Tuple2<Long, Integer>> testHarness = new StreamTaskTestHarness<Tuple2<Long, Integer>>(sourceTask, typeInfo);
        testHarness.setupOutputForSingletonOperatorChain();
        StreamConfig streamConfig = testHarness.getStreamConfig();
        StreamSource<Tuple2<Long, Integer>, ?> sourceOperator = new StreamSource<>(new MockSource(NUM_ELEMENTS, SOURCE_CHECKPOINT_DELAY, SOURCE_READ_DELAY));
        streamConfig.setStreamOperator(sourceOperator);
        // prepare the
        Future<Boolean>[] checkpointerResults = new Future[NUM_CHECKPOINTERS];
        // invoke this first, so the tasks are actually running when the checkpoints are scheduled
        testHarness.invoke();
        for (int i = 0; i < NUM_CHECKPOINTERS; i++) {
            checkpointerResults[i] = executor.submit(new Checkpointer(NUM_CHECKPOINTS, CHECKPOINT_INTERVAL, sourceTask));
        }
        testHarness.waitForTaskCompletion();
        // will be rethrown here
        for (int i = 0; i < NUM_CHECKPOINTERS; i++) {
            if (!checkpointerResults[i].isDone()) {
                checkpointerResults[i].cancel(true);
            }
            if (!checkpointerResults[i].isCancelled()) {
                checkpointerResults[i].get();
            }
        }
        List<Tuple2<Long, Integer>> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
        Assert.assertEquals(NUM_ELEMENTS, resultElements.size());
    } finally {
        executor.shutdown();
    }
}