Examples with OneInputStreamOperatorTestHarness org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness used on opensource projects

Example 31 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class ContinuousFileProcessingRescalingTest method getTestHarness.

private OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> getTestHarness(BlockingFileInputFormat format, int noOfTasks, int taksIdx) throws Exception {
    ContinuousFileReaderOperator<String> reader = new ContinuousFileReaderOperator<>(format);
    reader.setOutputType(TypeExtractor.getInputFormatTypes(format), new ExecutionConfig());
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness = new OneInputStreamOperatorTestHarness<>(reader, 10, noOfTasks, taksIdx);
    testHarness.setTimeCharacteristic(TimeCharacteristic.EventTime);
    return testHarness;
}

Example 32 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class StreamProjectTest method testProject.

@Test
public void testProject() throws Exception {
    TypeInformation<Tuple5<Integer, String, Integer, String, Integer>> inType = TypeExtractor.getForObject(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4));
    int[] fields = new int[] { 4, 4, 3 };
    TupleSerializer<Tuple3<Integer, Integer, String>> serializer = new TupleTypeInfo<Tuple3<Integer, Integer, String>>(StreamProjection.extractFieldTypes(fields, inType)).createSerializer(new ExecutionConfig());
    @SuppressWarnings("unchecked") StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> operator = new StreamProject<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(fields, serializer);
    OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>> testHarness = new OneInputStreamOperatorTestHarness<Tuple5<Integer, String, Integer, String, Integer>, Tuple3<Integer, Integer, String>>(operator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "b", 4), initialTime + 1));
    testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "s", 3, "c", 2), initialTime + 2));
    testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "c", 2), initialTime + 3));
    testHarness.processWatermark(new Watermark(initialTime + 2));
    testHarness.processElement(new StreamRecord<Tuple5<Integer, String, Integer, String, Integer>>(new Tuple5<Integer, String, Integer, String, Integer>(2, "a", 3, "a", 7), initialTime + 4));
    expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(4, 4, "b"), initialTime + 1));
    expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 2));
    expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(2, 2, "c"), initialTime + 3));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<Tuple3<Integer, Integer, String>>(new Tuple3<Integer, Integer, String>(7, 7, "a"), initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 33 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class AsyncWaitOperatorTest method testEventTime.

private void testEventTime(AsyncDataStream.OutputMode mode) throws Exception {
    final AsyncWaitOperator<Integer, Integer> operator = new AsyncWaitOperator<>(new MyAsyncFunction(), TIMEOUT, 2, mode);
    final OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator, IntSerializer.INSTANCE);
    final long initialTime = 0L;
    final ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    testHarness.open();
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.processElement(new StreamRecord<>(1, initialTime + 1));
        testHarness.processElement(new StreamRecord<>(2, initialTime + 2));
        testHarness.processWatermark(new Watermark(initialTime + 2));
        testHarness.processElement(new StreamRecord<>(3, initialTime + 3));
    }
    // wait until all async collectors in the buffer have been emitted out.
    synchronized (testHarness.getCheckpointLock()) {
        testHarness.close();
    }
    expectedOutput.add(new StreamRecord<>(2, initialTime + 1));
    expectedOutput.add(new StreamRecord<>(4, initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<>(6, initialTime + 3));
    if (AsyncDataStream.OutputMode.ORDERED == mode) {
        TestHarnessUtil.assertOutputEquals("Output with watermark was not correct.", expectedOutput, testHarness.getOutput());
    } else {
        Object[] jobOutputQueue = testHarness.getOutput().toArray();
        Assert.assertEquals("Watermark should be at index 2", new Watermark(initialTime + 2), jobOutputQueue[2]);
        Assert.assertEquals("StreamRecord 3 should be at the end", new StreamRecord<>(6, initialTime + 3), jobOutputQueue[3]);
        TestHarnessUtil.assertOutputEqualsSorted("Output for StreamRecords does not match", expectedOutput, testHarness.getOutput(), new StreamRecordComparator());
    }
}

Example 34 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class StreamFilterTest method testFilter.

@Test
@SuppressWarnings("unchecked")
public void testFilter() throws Exception {
    StreamFilter<Integer> operator = new StreamFilter<Integer>(new MyFilter());
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
    testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
    testHarness.processWatermark(new Watermark(initialTime + 2));
    testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
    testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
    testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
    testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
    testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));
    expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
    expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 35 with OneInputStreamOperatorTestHarness

use of org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness in project flink by apache.

the class StreamFlatMapTest method testFlatMap.

@Test
public void testFlatMap() throws Exception {
    StreamFlatMap<Integer, Integer> operator = new StreamFlatMap<Integer, Integer>(new MyFlatMap());
    OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<Integer, Integer>(operator);
    long initialTime = 0L;
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    testHarness.open();
    testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
    testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
    testHarness.processWatermark(new Watermark(initialTime + 2));
    testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));
    testHarness.processElement(new StreamRecord<Integer>(4, initialTime + 4));
    testHarness.processElement(new StreamRecord<Integer>(5, initialTime + 5));
    testHarness.processElement(new StreamRecord<Integer>(6, initialTime + 6));
    testHarness.processElement(new StreamRecord<Integer>(7, initialTime + 7));
    testHarness.processElement(new StreamRecord<Integer>(8, initialTime + 8));
    expectedOutput.add(new StreamRecord<Integer>(2, initialTime + 2));
    expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 2));
    expectedOutput.add(new Watermark(initialTime + 2));
    expectedOutput.add(new StreamRecord<Integer>(4, initialTime + 4));
    expectedOutput.add(new StreamRecord<Integer>(16, initialTime + 4));
    expectedOutput.add(new StreamRecord<Integer>(6, initialTime + 6));
    expectedOutput.add(new StreamRecord<Integer>(36, initialTime + 6));
    expectedOutput.add(new StreamRecord<Integer>(8, initialTime + 8));
    expectedOutput.add(new StreamRecord<Integer>(64, initialTime + 8));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}