Examples with StreamOperator org.apache.flink.streaming.api.operators.StreamOperator used on opensource projects

Example 1 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class BufferDataOverWindowOperatorTest method test.

private void test(OverWindowFrame[] frames, GenericRowData[] expect) throws Exception {
    MockEnvironment env = new MockEnvironmentBuilder().setIOManager(ioManager).setMemoryManager(memoryManager).build();
    StreamTask<Object, StreamOperator<Object>> task = new StreamTask<Object, StreamOperator<Object>>(env) {

        @Override
        protected void init() {
        }
    };
    operator = new BufferDataOverWindowOperator(frames, comparator, true) {

        {
            output = new NonBufferOverWindowOperatorTest.ConsumerOutput(new Consumer<RowData>() {

                @Override
                public void accept(RowData r) {
                    collect.add(GenericRowData.of(r.getInt(0), r.getLong(1), r.getLong(2), r.getLong(3), r.getLong(4)));
                }
            });
        }

        @Override
        public ClassLoader getUserCodeClassloader() {
            return Thread.currentThread().getContextClassLoader();
        }

        @Override
        public StreamConfig getOperatorConfig() {
            StreamConfig conf = mock(StreamConfig.class);
            when(conf.<RowData>getTypeSerializerIn1(getUserCodeClassloader())).thenReturn(inputSer);
            when(conf.getManagedMemoryFractionOperatorUseCaseOfSlot(eq(ManagedMemoryUseCase.OPERATOR), any(Configuration.class), any(ClassLoader.class))).thenReturn(0.99);
            return conf;
        }

        @Override
        public StreamTask<?, ?> getContainingTask() {
            return task;
        }

        @Override
        public StreamingRuntimeContext getRuntimeContext() {
            return mock(StreamingRuntimeContext.class);
        }
    };
    operator.setProcessingTimeService(new TestProcessingTimeService());
    operator.open();
    addRow(0, 1L, 4L);
    /* 1 **/
    addRow(0, 1L, 1L);
    /* 2 **/
    addRow(0, 1L, 1L);
    /* 3 **/
    addRow(0, 1L, 1L);
    /* 4 **/
    addRow(1, 5L, 2L);
    /* 5 **/
    addRow(2, 5L, 4L);
    /* 6 **/
    addRow(2, 6L, 2L);
    /* 7 **/
    addRow(2, 6L, 2L);
    /* 8 **/
    addRow(2, 6L, 2L);
    /* 9 **/
    operator.endInput();
    GenericRowData[] outputs = this.collect.toArray(new GenericRowData[0]);
    Assert.assertArrayEquals(expect, outputs);
    operator.close();
}

Example 2 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class String2SortMergeJoinOperatorTest method testFullJoin.

@Test
public void testFullJoin() throws Exception {
    StreamOperator joinOperator = newOperator(FlinkJoinType.FULL, leftIsSmall);
    TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = buildSortMergeJoin(joinOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
    expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
    expectedOutput.add(new StreamRecord<>(newRow("c", "2null")));
    expectedOutput.add(new StreamRecord<>(newRow("d", "0null")));
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput()));
}

Example 3 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class String2SortMergeJoinOperatorTest method testLeftOuterJoin.

@Test
public void testLeftOuterJoin() throws Exception {
    StreamOperator joinOperator = newOperator(FlinkJoinType.LEFT, leftIsSmall);
    TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = buildSortMergeJoin(joinOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
    expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
    expectedOutput.add(new StreamRecord<>(newRow("d", "0null")));
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput()));
}

Example 4 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class Int2SortMergeJoinOperatorTest method testSemiJoin.

@Test
public void testSemiJoin() throws Exception {
    int numKeys1 = 10;
    int numKeys2 = 9;
    int buildValsPerKey = 10;
    int probeValsPerKey = 3;
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys1, buildValsPerKey, true);
    MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys2, probeValsPerKey, true);
    StreamOperator operator = newOperator(FlinkJoinType.SEMI, false);
    joinAndAssert(operator, buildInput, probeInput, 90, 9, 45, true);
}

Example 5 with StreamOperator

use of org.apache.flink.streaming.api.operators.StreamOperator in project flink by apache.

the class Int2SortMergeJoinOperatorTest method testAntiJoin.

@Test
public void testAntiJoin() throws Exception {
    int numKeys1 = 10;
    int numKeys2 = 9;
    int buildValsPerKey = 10;
    int probeValsPerKey = 3;
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys1, buildValsPerKey, true);
    MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys2, probeValsPerKey, true);
    StreamOperator operator = newOperator(FlinkJoinType.ANTI, false);
    joinAndAssert(operator, buildInput, probeInput, 10, 1, 45, true);
}