Examples with StreamSource org.apache.flink.streaming.api.operators.StreamSource used on opensource projects

Example 6 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class StreamSourceOperatorTest method testLatencyMarkEmission.

/**
	 * Test that latency marks are emitted
	 */
@Test
public void testLatencyMarkEmission() throws Exception {
    final List<StreamElement> output = new ArrayList<>();
    final long maxProcessingTime = 100L;
    final long latencyMarkInterval = 10L;
    final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
    testProcessingTimeService.setCurrentTime(0L);
    final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
    // regular stream source operator
    final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));
    // emit latency marks every 10 milliseconds.
    setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, latencyMarkInterval, testProcessingTimeService);
    // run and wait to be stopped
    operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));
    int numberLatencyMarkers = (int) (maxProcessingTime / latencyMarkInterval) + 1;
    assertEquals(// + 1 is the final watermark element
    numberLatencyMarkers + 1, output.size());
    long timestamp = 0L;
    int i = 0;
    // and that its only latency markers + a final watermark
    for (; i < output.size() - 1; i++) {
        StreamElement se = output.get(i);
        Assert.assertTrue(se.isLatencyMarker());
        Assert.assertEquals(-1, se.asLatencyMarker().getVertexID());
        Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
        Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp);
        timestamp += latencyMarkInterval;
    }
    Assert.assertTrue(output.get(i).isWatermark());
}

Example 7 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class SourceStreamTaskTest method testOpenClose.

/**
	 * This test verifies that open() and close() are correctly called by the StreamTask.
	 */
@Test
public void testOpenClose() throws Exception {
    final SourceStreamTask<String, SourceFunction<String>, StreamSource<String, SourceFunction<String>>> sourceTask = new SourceStreamTask<>();
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<String>(sourceTask, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
    streamConfig.setStreamOperator(sourceOperator);
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    Assert.assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(10, resultElements.size());
}

Example 8 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class FlinkKafkaConsumerBaseMigrationTest method testRestoreFromFlink11WithEmptyStateWithPartitions.

/** Test restoring from an empty state taken using Flink 1.1, when some partitions could be found for topics. */
@Test
public void testRestoreFromFlink11WithEmptyStateWithPartitions() throws Exception {
    final List<KafkaTopicPartition> partitions = new ArrayList<>();
    partitions.add(new KafkaTopicPartition("abc", 13));
    partitions.add(new KafkaTopicPartition("def", 7));
    final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
    StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
    final AbstractStreamOperatorTestHarness<String> testHarness = new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
    testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.setup();
    // restore state from binary snapshot file using legacy method
    testHarness.initializeStateFromLegacyCheckpoint(getResourceFilename("kafka-consumer-migration-test-flink1.1-snapshot-empty-state"));
    testHarness.open();
    // the expected state in "kafka-consumer-migration-test-flink1.1-snapshot-empty-state";
    // since the state is empty, the consumer should reflect on the startup mode to determine start offsets.
    final HashMap<KafkaTopicPartition, Long> expectedSubscribedPartitionsWithStartOffsets = new HashMap<>();
    expectedSubscribedPartitionsWithStartOffsets.put(new KafkaTopicPartition("abc", 13), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    expectedSubscribedPartitionsWithStartOffsets.put(new KafkaTopicPartition("def", 7), KafkaTopicPartitionStateSentinel.GROUP_OFFSET);
    // assert that there are partitions and is identical to expected list
    Assert.assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
    Assert.assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
    Assert.assertEquals(expectedSubscribedPartitionsWithStartOffsets, consumerFunction.getSubscribedPartitionsToStartOffsets());
    // assert that no state was restored
    Assert.assertTrue(consumerFunction.getRestoredState() == null);
    consumerOperator.close();
    consumerOperator.cancel();
}

Example 9 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class FromElementsFunctionTest method testCheckpointAndRestore.

@Test
public void testCheckpointAndRestore() {
    try {
        final int NUM_ELEMENTS = 10000;
        List<Integer> data = new ArrayList<Integer>(NUM_ELEMENTS);
        List<Integer> result = new ArrayList<Integer>(NUM_ELEMENTS);
        for (int i = 0; i < NUM_ELEMENTS; i++) {
            data.add(i);
        }
        final FromElementsFunction<Integer> source = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
        StreamSource<Integer, FromElementsFunction<Integer>> src = new StreamSource<>(source);
        AbstractStreamOperatorTestHarness<Integer> testHarness = new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
        testHarness.open();
        final SourceFunction.SourceContext<Integer> ctx = new ListSourceContext<Integer>(result, 2L);
        final Throwable[] error = new Throwable[1];
        // run the source asynchronously
        Thread runner = new Thread() {

            @Override
            public void run() {
                try {
                    source.run(ctx);
                } catch (Throwable t) {
                    error[0] = t;
                }
            }
        };
        runner.start();
        // wait for a bit 
        Thread.sleep(1000);
        // make a checkpoint
        List<Integer> checkpointData = new ArrayList<>(NUM_ELEMENTS);
        OperatorStateHandles handles = null;
        synchronized (ctx.getCheckpointLock()) {
            handles = testHarness.snapshot(566, System.currentTimeMillis());
            checkpointData.addAll(result);
        }
        // cancel the source
        source.cancel();
        runner.join();
        // check for errors
        if (error[0] != null) {
            System.err.println("Error in asynchronous source runner");
            error[0].printStackTrace();
            fail("Error in asynchronous source runner");
        }
        final FromElementsFunction<Integer> sourceCopy = new FromElementsFunction<>(IntSerializer.INSTANCE, data);
        StreamSource<Integer, FromElementsFunction<Integer>> srcCopy = new StreamSource<>(sourceCopy);
        AbstractStreamOperatorTestHarness<Integer> testHarnessCopy = new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
        testHarnessCopy.setup();
        testHarnessCopy.initializeState(handles);
        testHarnessCopy.open();
        // recovery run
        SourceFunction.SourceContext<Integer> newCtx = new ListSourceContext<>(checkpointData);
        sourceCopy.run(newCtx);
        assertEquals(data, checkpointData);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class StatefulSequenceSourceTest method testCheckpointRestore.

@Test
public void testCheckpointRestore() throws Exception {
    final int initElement = 0;
    final int maxElement = 100;
    final Set<Long> expectedOutput = new HashSet<>();
    for (long i = initElement; i <= maxElement; i++) {
        expectedOutput.add(i);
    }
    final ConcurrentHashMap<String, List<Long>> outputCollector = new ConcurrentHashMap<>();
    final OneShotLatch latchToTrigger1 = new OneShotLatch();
    final OneShotLatch latchToWait1 = new OneShotLatch();
    final OneShotLatch latchToTrigger2 = new OneShotLatch();
    final OneShotLatch latchToWait2 = new OneShotLatch();
    final StatefulSequenceSource source1 = new StatefulSequenceSource(initElement, maxElement);
    StreamSource<Long, StatefulSequenceSource> src1 = new StreamSource<>(source1);
    final AbstractStreamOperatorTestHarness<Long> testHarness1 = new AbstractStreamOperatorTestHarness<>(src1, 2, 2, 0);
    testHarness1.open();
    final StatefulSequenceSource source2 = new StatefulSequenceSource(initElement, maxElement);
    StreamSource<Long, StatefulSequenceSource> src2 = new StreamSource<>(source2);
    final AbstractStreamOperatorTestHarness<Long> testHarness2 = new AbstractStreamOperatorTestHarness<>(src2, 2, 2, 1);
    testHarness2.open();
    final Throwable[] error = new Throwable[3];
    // run the source asynchronously
    Thread runner1 = new Thread() {

        @Override
        public void run() {
            try {
                source1.run(new BlockingSourceContext("1", latchToTrigger1, latchToWait1, outputCollector, 21));
            } catch (Throwable t) {
                t.printStackTrace();
                error[0] = t;
            }
        }
    };
    // run the source asynchronously
    Thread runner2 = new Thread() {

        @Override
        public void run() {
            try {
                source2.run(new BlockingSourceContext("2", latchToTrigger2, latchToWait2, outputCollector, 32));
            } catch (Throwable t) {
                t.printStackTrace();
                error[1] = t;
            }
        }
    };
    runner1.start();
    runner2.start();
    if (!latchToTrigger1.isTriggered()) {
        latchToTrigger1.await();
    }
    if (!latchToTrigger2.isTriggered()) {
        latchToTrigger2.await();
    }
    OperatorStateHandles snapshot = AbstractStreamOperatorTestHarness.repackageState(testHarness1.snapshot(0L, 0L), testHarness2.snapshot(0L, 0L));
    final StatefulSequenceSource source3 = new StatefulSequenceSource(initElement, maxElement);
    StreamSource<Long, StatefulSequenceSource> src3 = new StreamSource<>(source3);
    final AbstractStreamOperatorTestHarness<Long> testHarness3 = new AbstractStreamOperatorTestHarness<>(src3, 2, 1, 0);
    testHarness3.setup();
    testHarness3.initializeState(snapshot);
    testHarness3.open();
    final OneShotLatch latchToTrigger3 = new OneShotLatch();
    final OneShotLatch latchToWait3 = new OneShotLatch();
    latchToWait3.trigger();
    // run the source asynchronously
    Thread runner3 = new Thread() {

        @Override
        public void run() {
            try {
                source3.run(new BlockingSourceContext("3", latchToTrigger3, latchToWait3, outputCollector, 3));
            } catch (Throwable t) {
                t.printStackTrace();
                error[2] = t;
            }
        }
    };
    runner3.start();
    runner3.join();
    // we have 3 tasks.
    Assert.assertEquals(3, outputCollector.size());
    // test for at-most-once
    Set<Long> dedupRes = new HashSet<>(Math.abs(maxElement - initElement) + 1);
    for (Map.Entry<String, List<Long>> elementsPerTask : outputCollector.entrySet()) {
        String key = elementsPerTask.getKey();
        List<Long> elements = outputCollector.get(key);
        // this tests the correctness of the latches in the test
        Assert.assertTrue(elements.size() > 0);
        for (Long elem : elements) {
            if (!dedupRes.add(elem)) {
                Assert.fail("Duplicate entry: " + elem);
            }
            if (!expectedOutput.contains(elem)) {
                Assert.fail("Unexpected element: " + elem);
            }
        }
    }
    // test for exactly-once
    Assert.assertEquals(Math.abs(initElement - maxElement) + 1, dedupRes.size());
    latchToWait1.trigger();
    latchToWait2.trigger();
    // wait for everybody ot finish.
    runner1.join();
    runner2.join();
}