Examples with StreamSource org.apache.flink.streaming.api.operators.StreamSource used on opensource projects

Example 1 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class SourceStreamTaskTest method testOpenClose.

/**
	 * This test verifies that open() and close() are correctly called by the StreamTask.
	 */
@Test
public void testOpenClose() throws Exception {
    final SourceStreamTask<String, SourceFunction<String>, StreamSource<String, SourceFunction<String>>> sourceTask = new SourceStreamTask<>();
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<String>(sourceTask, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
    streamConfig.setStreamOperator(sourceOperator);
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    Assert.assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(10, resultElements.size());
}

Example 2 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class StreamGraph method addOperator.

public <IN, OUT> void addOperator(Integer vertexID, String slotSharingGroup, StreamOperator<OUT> operatorObject, TypeInformation<IN> inTypeInfo, TypeInformation<OUT> outTypeInfo, String operatorName) {
    if (operatorObject instanceof StoppableStreamSource) {
        addNode(vertexID, slotSharingGroup, StoppableSourceStreamTask.class, operatorObject, operatorName);
    } else if (operatorObject instanceof StreamSource) {
        addNode(vertexID, slotSharingGroup, SourceStreamTask.class, operatorObject, operatorName);
    } else {
        addNode(vertexID, slotSharingGroup, OneInputStreamTask.class, operatorObject, operatorName);
    }
    TypeSerializer<IN> inSerializer = inTypeInfo != null && !(inTypeInfo instanceof MissingTypeInfo) ? inTypeInfo.createSerializer(executionConfig) : null;
    TypeSerializer<OUT> outSerializer = outTypeInfo != null && !(outTypeInfo instanceof MissingTypeInfo) ? outTypeInfo.createSerializer(executionConfig) : null;
    setSerializers(vertexID, inSerializer, null, outSerializer);
    if (operatorObject instanceof OutputTypeConfigurable && outTypeInfo != null) {
        @SuppressWarnings("unchecked") OutputTypeConfigurable<OUT> outputTypeConfigurable = (OutputTypeConfigurable<OUT>) operatorObject;
        // sets the output type which must be know at StreamGraph creation time
        outputTypeConfigurable.setOutputType(outTypeInfo, executionConfig);
    }
    if (operatorObject instanceof InputTypeConfigurable) {
        InputTypeConfigurable inputTypeConfigurable = (InputTypeConfigurable) operatorObject;
        inputTypeConfigurable.setInputType(inTypeInfo, executionConfig);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Vertex: {}", vertexID);
    }
}

Example 3 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class StreamSourceOperatorTest method testLatencyMarkEmission.

/**
	 * Test that latency marks are emitted
	 */
@Test
public void testLatencyMarkEmission() throws Exception {
    final List<StreamElement> output = new ArrayList<>();
    final long maxProcessingTime = 100L;
    final long latencyMarkInterval = 10L;
    final TestProcessingTimeService testProcessingTimeService = new TestProcessingTimeService();
    testProcessingTimeService.setCurrentTime(0L);
    final List<Long> processingTimes = Arrays.asList(1L, 10L, 11L, 21L, maxProcessingTime);
    // regular stream source operator
    final StreamSource<Long, ProcessingTimeServiceSource> operator = new StreamSource<>(new ProcessingTimeServiceSource(testProcessingTimeService, processingTimes));
    // emit latency marks every 10 milliseconds.
    setupSourceOperator(operator, TimeCharacteristic.EventTime, 0, latencyMarkInterval, testProcessingTimeService);
    // run and wait to be stopped
    operator.run(new Object(), mock(StreamStatusMaintainer.class), new CollectorOutput<Long>(output));
    int numberLatencyMarkers = (int) (maxProcessingTime / latencyMarkInterval) + 1;
    assertEquals(// + 1 is the final watermark element
    numberLatencyMarkers + 1, output.size());
    long timestamp = 0L;
    int i = 0;
    // and that its only latency markers + a final watermark
    for (; i < output.size() - 1; i++) {
        StreamElement se = output.get(i);
        Assert.assertTrue(se.isLatencyMarker());
        Assert.assertEquals(-1, se.asLatencyMarker().getVertexID());
        Assert.assertEquals(0, se.asLatencyMarker().getSubtaskIndex());
        Assert.assertTrue(se.asLatencyMarker().getMarkedTime() == timestamp);
        timestamp += latencyMarkInterval;
    }
    Assert.assertTrue(output.get(i).isWatermark());
}

Example 4 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class FlinkKafkaConsumerBaseMigrationTest method testRestoreFromFlink11.

/** Test restoring from a non-empty state taken using Flink 1.1, when some partitions could be found for topics. */
@Test
public void testRestoreFromFlink11() throws Exception {
    final List<KafkaTopicPartition> partitions = new ArrayList<>();
    partitions.add(new KafkaTopicPartition("abc", 13));
    partitions.add(new KafkaTopicPartition("def", 7));
    final DummyFlinkKafkaConsumer<String> consumerFunction = new DummyFlinkKafkaConsumer<>(partitions);
    StreamSource<String, DummyFlinkKafkaConsumer<String>> consumerOperator = new StreamSource<>(consumerFunction);
    final AbstractStreamOperatorTestHarness<String> testHarness = new AbstractStreamOperatorTestHarness<>(consumerOperator, 1, 1, 0);
    testHarness.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.setup();
    // restore state from binary snapshot file using legacy method
    testHarness.initializeStateFromLegacyCheckpoint(getResourceFilename("kafka-consumer-migration-test-flink1.1-snapshot"));
    testHarness.open();
    // the expected state in "kafka-consumer-migration-test-flink1.1-snapshot"
    final HashMap<KafkaTopicPartition, Long> expectedState = new HashMap<>();
    expectedState.put(new KafkaTopicPartition("abc", 13), 16768L);
    expectedState.put(new KafkaTopicPartition("def", 7), 987654321L);
    // assert that there are partitions and is identical to expected list
    Assert.assertTrue(consumerFunction.getSubscribedPartitionsToStartOffsets() != null);
    Assert.assertTrue(!consumerFunction.getSubscribedPartitionsToStartOffsets().isEmpty());
    // on restore, subscribedPartitionsToStartOffsets should be identical to the restored state
    Assert.assertEquals(expectedState, consumerFunction.getSubscribedPartitionsToStartOffsets());
    // assert that state is correctly restored from legacy checkpoint
    Assert.assertTrue(consumerFunction.getRestoredState() != null);
    Assert.assertEquals(expectedState, consumerFunction.getRestoredState());
    consumerOperator.close();
    consumerOperator.cancel();
}

Example 5 with StreamSource

use of org.apache.flink.streaming.api.operators.StreamSource in project flink by apache.

the class RMQSourceTest method testCheckpointing.

@Test
public void testCheckpointing() throws Exception {
    source.autoAck = false;
    StreamSource<String, RMQSource<String>> src = new StreamSource<>(source);
    AbstractStreamOperatorTestHarness<String> testHarness = new AbstractStreamOperatorTestHarness<>(src, 1, 1, 0);
    testHarness.open();
    sourceThread.start();
    Thread.sleep(5);
    final Random random = new Random(System.currentTimeMillis());
    int numSnapshots = 50;
    long previousSnapshotId;
    long lastSnapshotId = 0;
    long totalNumberOfAcks = 0;
    for (int i = 0; i < numSnapshots; i++) {
        long snapshotId = random.nextLong();
        OperatorStateHandles data;
        synchronized (DummySourceContext.lock) {
            data = testHarness.snapshot(snapshotId, System.currentTimeMillis());
            previousSnapshotId = lastSnapshotId;
            lastSnapshotId = messageId;
        }
        // let some time pass
        Thread.sleep(5);
        // check if the correct number of messages have been snapshotted
        final long numIds = lastSnapshotId - previousSnapshotId;
        RMQTestSource sourceCopy = new RMQTestSource();
        StreamSource<String, RMQTestSource> srcCopy = new StreamSource<>(sourceCopy);
        AbstractStreamOperatorTestHarness<String> testHarnessCopy = new AbstractStreamOperatorTestHarness<>(srcCopy, 1, 1, 0);
        testHarnessCopy.setup();
        testHarnessCopy.initializeState(data);
        testHarnessCopy.open();
        ArrayDeque<Tuple2<Long, List<String>>> deque = sourceCopy.getRestoredState();
        List<String> messageIds = deque.getLast().f1;
        assertEquals(numIds, messageIds.size());
        if (messageIds.size() > 0) {
            assertEquals(lastSnapshotId, (long) Long.valueOf(messageIds.get(messageIds.size() - 1)));
        }
        // check if the messages are being acknowledged and the transaction committed
        synchronized (DummySourceContext.lock) {
            source.notifyCheckpointComplete(snapshotId);
        }
        totalNumberOfAcks += numIds;
    }
    Mockito.verify(source.channel, Mockito.times((int) totalNumberOfAcks)).basicAck(Mockito.anyLong(), Mockito.eq(false));
    Mockito.verify(source.channel, Mockito.times(numSnapshots)).txCommit();
}