Examples with BinaryRowData org.apache.flink.table.data.binary.BinaryRowData used on opensource projects

Example 21 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class SortMergeJoinIteratorTest method inner.

public void inner(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator input1 = data.f0;
    MutableObjectIterator input2 = data.f1;
    if (leftIsSmall) {
        input1 = data.f1;
        input2 = data.f0;
    }
    try (SortMergeInnerJoinIterator iterator = new SortMergeInnerJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextInnerJoin()) {
            RowData probe = iterator.getProbeRow();
            ResettableExternalBuffer.BufferIterator iter = iterator.getMatchBuffer().newIterator();
            while (iter.advanceNext()) {
                RowData row = iter.getRow();
                Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                if (leftIsSmall) {
                    assertEquals(expected, new Tuple2<>(row, probe));
                } else {
                    assertEquals(expected, new Tuple2<>(probe, row));
                }
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 22 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class SortMergeJoinIteratorTest method fullOuter.

public void fullOuter(Tuple2<MutableObjectIterator<BinaryRowData>, MutableObjectIterator<BinaryRowData>> data, List<Tuple2<BinaryRowData, BinaryRowData>> compare) throws Exception {
    MutableObjectIterator<BinaryRowData> input1 = data.f0;
    MutableObjectIterator<BinaryRowData> input2 = data.f1;
    try (SortMergeFullOuterJoinIterator iterator = new SortMergeFullOuterJoinIterator(new BinaryRowDataSerializer(1), new BinaryRowDataSerializer(1), new MyProjection(), new MyProjection(), new IntRecordComparator(), input1, input2, new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new ResettableExternalBuffer(ioManager, new LazyMemorySegmentPool(this, memManager, BUFFER_MEMORY), serializer, false), new boolean[] { true })) {
        int id = 0;
        while (iterator.nextOuterJoin()) {
            BinaryRowData matchKey = iterator.getMatchKey();
            ResettableExternalBuffer buffer1 = iterator.getBuffer1();
            ResettableExternalBuffer buffer2 = iterator.getBuffer2();
            if (matchKey == null && buffer1.size() > 0) {
                // left outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer1.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(row, null));
                }
            } else if (matchKey == null && buffer2.size() > 0) {
                // right outer join.
                ResettableExternalBuffer.BufferIterator iter = buffer2.newIterator();
                while (iter.advanceNext()) {
                    RowData row = iter.getRow();
                    Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                    assertEquals(expected, new Tuple2<>(null, row));
                }
            } else if (matchKey != null) {
                // match join.
                ResettableExternalBuffer.BufferIterator iter1 = buffer1.newIterator();
                while (iter1.advanceNext()) {
                    RowData row1 = iter1.getRow();
                    ResettableExternalBuffer.BufferIterator iter2 = buffer2.newIterator();
                    while (iter2.advanceNext()) {
                        RowData row2 = iter2.getRow();
                        Tuple2<BinaryRowData, BinaryRowData> expected = compare.get(id++);
                        assertEquals(expected, new Tuple2<>(row1, row2));
                    }
                }
            } else {
                // bug...
                throw new RuntimeException("There is a bug.");
            }
        }
        assertEquals(compare.size(), id);
    }
}

Example 23 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class String2SortMergeJoinOperatorTest method testFullJoin.

@Test
public void testFullJoin() throws Exception {
    StreamOperator joinOperator = newOperator(FlinkJoinType.FULL, leftIsSmall);
    TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = buildSortMergeJoin(joinOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
    expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
    expectedOutput.add(new StreamRecord<>(newRow("c", "2null")));
    expectedOutput.add(new StreamRecord<>(newRow("d", "0null")));
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput()));
}

Example 24 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class String2SortMergeJoinOperatorTest method testLeftOuterJoin.

@Test
public void testLeftOuterJoin() throws Exception {
    StreamOperator joinOperator = newOperator(FlinkJoinType.LEFT, leftIsSmall);
    TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = buildSortMergeJoin(joinOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new StreamRecord<>(newRow("a", "02")));
    expectedOutput.add(new StreamRecord<>(newRow("b", "14")));
    expectedOutput.add(new StreamRecord<>(newRow("d", "0null")));
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, transformToBinary(testHarness.getOutput()));
}

Example 25 with BinaryRowData

use of org.apache.flink.table.data.binary.BinaryRowData in project flink by apache.

the class String2SortMergeJoinOperatorTest method buildSortMergeJoin.

private TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> buildSortMergeJoin(StreamOperator operator) throws Exception {
    final TwoInputStreamTaskTestHarness<BinaryRowData, BinaryRowData, JoinedRowData> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2, new int[] { 1, 2 }, typeInfo, (TypeInformation) typeInfo, joinedInfo);
    testHarness.memorySize = 36 * 1024 * 1024;
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.getStreamConfig().setStreamOperator(operator);
    testHarness.getStreamConfig().setOperatorID(new OperatorID());
    testHarness.getStreamConfig().setManagedMemoryFractionOperatorOfUseCase(ManagedMemoryUseCase.OPERATOR, 0.99);
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new StreamRecord<>(newRow("a", "0"), initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<>(newRow("d", "0"), initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<>(newRow("a", "2"), initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(newRow("b", "1"), initialTime), 0, 1);
    testHarness.processElement(new StreamRecord<>(newRow("c", "2"), initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(newRow("b", "4"), initialTime), 1, 0);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    return testHarness;
}